From d9a987405fa17cfe9f081d26c467097407af02a1 Mon Sep 17 00:00:00 2001 From: Stefano Ortolani Date: Fri, 28 Feb 2025 23:10:29 +0000 Subject: [PATCH] Refactor and move all dependencies to the 'all' extra --- .github/workflows/release-package.yml | 2 +- .github/workflows/test-package.yml | 10 +- .pre-commit-config.yaml | 43 + Makefile | 10 +- README.md | 3 +- documentation/README.md | 47 +- documentation/generate.py | 179 +++ documentation/generate_documentation.py | 178 --- documentation/mkdocs/expansion.md | 27 +- documentation/mkdocs/export_mod.md | 10 + documentation/mkdocs/import_mod.md | 10 + documentation/mkdocs/index.md | 3 +- misp_modules/__init__.py | 382 +---- misp_modules/__main__.py | 260 +++ misp_modules/helpers/__init__.py | 1 - misp_modules/helpers/cache.py | 22 +- misp_modules/lib/ODTReader/LICENSE | 674 ++++++++ misp_modules/lib/ODTReader/__init__.py | 0 misp_modules/lib/ODTReader/odtreader.py | 57 + .../lib/ODTReader/win32_unicode_argv.py | 52 + misp_modules/lib/__init__.py | 4 - misp_modules/lib/_vmray/parser.py | 97 +- misp_modules/lib/_vmray/rest_api.py | 50 +- misp_modules/lib/cof2misp/cof.py | 71 +- misp_modules/lib/dnstrails/LICENSE | 21 + misp_modules/lib/dnstrails/__init__.py | 2 + misp_modules/lib/dnstrails/api.py | 424 +++++ misp_modules/lib/dnstrails/exception.py | 7 + misp_modules/lib/joe_mapping.py | 152 +- misp_modules/lib/joe_parser.py | 534 ++++--- misp_modules/lib/lastline_api.py | 136 +- misp_modules/lib/onyphe/LICENSE | 21 + misp_modules/lib/onyphe/__init__.py | 2 + misp_modules/lib/onyphe/client.py | 701 +++++++++ misp_modules/lib/onyphe/exception.py | 16 + misp_modules/lib/qintel_helper.py | 156 +- misp_modules/lib/vt_graph_parser/__init__.py | 1 - .../lib/vt_graph_parser/helpers/__init__.py | 1 - .../lib/vt_graph_parser/helpers/parsers.py | 25 +- .../lib/vt_graph_parser/helpers/rules.py | 13 +- .../lib/vt_graph_parser/helpers/wrappers.py | 8 +- .../lib/vt_graph_parser/importers/__init__.py | 1 - .../lib/vt_graph_parser/importers/base.py | 58 +- .../importers/pymisp_response.py | 41 +- misp_modules/modules/__init__.py | 4 - misp_modules/modules/action_mod/__init__.py | 1 - .../modules/action_mod/_utils/__init__.py | 1 - .../modules/action_mod/_utils/utils.py | 2 +- misp_modules/modules/action_mod/mattermost.py | 109 +- misp_modules/modules/action_mod/slack.py | 78 +- misp_modules/modules/action_mod/testaction.py | 52 +- misp_modules/modules/expansion/__init__.py | 123 +- .../expansion/_dnsdb_query/dnsdb_query.py | 248 +-- .../expansion/_ransomcoindb/ransomcoindb.py | 72 +- .../vulnerability_parser.py | 458 +++--- misp_modules/modules/expansion/abuseipdb.py | 149 +- misp_modules/modules/expansion/apiosintds.py | 674 ++++++-- misp_modules/modules/expansion/apivoid.py | 197 ++- .../modules/expansion/assemblyline_query.py | 171 +- .../modules/expansion/assemblyline_submit.py | 89 +- .../modules/expansion/backscatter_io.py | 83 +- .../modules/expansion/btc_scam_check.py | 61 +- .../modules/expansion/btc_steroids.py | 153 +- .../modules/expansion/censys_enrich.py | 276 ++-- .../modules/expansion/circl_passivedns.py | 99 +- .../modules/expansion/circl_passivessl.py | 110 +- misp_modules/modules/expansion/clamav.py | 47 +- .../modules/expansion/cluster25_expand.py | 242 +-- .../expansion/convert_markdown_to_pdf.py | 105 +- misp_modules/modules/expansion/countrycode.py | 59 +- misp_modules/modules/expansion/cpe.py | 147 +- misp_modules/modules/expansion/crowdsec.py | 79 +- .../modules/expansion/crowdstrike_falcon.py | 190 ++- .../modules/expansion/cuckoo_submit.py | 61 +- misp_modules/modules/expansion/cve.py | 52 +- .../modules/expansion/cve_advanced.py | 142 +- .../modules/expansion/cytomic_orion.py | 307 ++-- .../modules/expansion/dbl_spamhaus.py | 93 +- misp_modules/modules/expansion/dns.py | 72 +- misp_modules/modules/expansion/docx_enrich.py | 63 +- misp_modules/modules/expansion/domaintools.py | 321 ++-- misp_modules/modules/expansion/eql.py | 48 +- misp_modules/modules/expansion/eupi.py | 83 +- .../expansion/extract_url_components.py | 87 +- .../modules/expansion/farsight_passivedns.py | 260 +-- misp_modules/modules/expansion/geoip_asn.py | 75 +- misp_modules/modules/expansion/geoip_city.py | 80 +- .../modules/expansion/geoip_country.py | 67 +- .../modules/expansion/google_safe_browsing.py | 88 +- .../modules/expansion/google_search.py | 57 - .../expansion/google_threat_intelligence.py | 409 +++-- misp_modules/modules/expansion/greynoise.py | 63 +- misp_modules/modules/expansion/hashdd.py | 46 +- misp_modules/modules/expansion/hashlookup.py | 127 +- misp_modules/modules/expansion/hibp.py | 59 +- .../modules/expansion/html_to_markdown.py | 47 +- misp_modules/modules/expansion/hyasinsight.py | 969 ++++++------ misp_modules/modules/expansion/intel471.py | 98 +- .../modules/expansion/ip2locationio.py | 95 +- misp_modules/modules/expansion/ipasn.py | 71 +- misp_modules/modules/expansion/ipinfo.py | 126 +- .../expansion/ipqs_fraud_and_risk_scoring.py | 540 ++++--- misp_modules/modules/expansion/iprep.py | 129 +- .../expansion/jinja_template_rendering.py | 44 +- .../modules/expansion/joesandbox_query.py | 95 +- .../modules/expansion/joesandbox_submit.py | 59 +- .../modules/expansion/lastline_query.py | 65 +- .../modules/expansion/lastline_submit.py | 31 +- .../modules/expansion/macaddress_io.py | 127 +- misp_modules/modules/expansion/macvendors.py | 60 +- .../modules/expansion/malshare_upload.py | 60 +- .../modules/expansion/malwarebazaar.py | 86 +- .../expansion/mcafee_insights_enrich.py | 184 +-- misp_modules/modules/expansion/mmdb_lookup.py | 204 ++- misp_modules/modules/expansion/mwdb.py | 82 +- misp_modules/modules/expansion/ocr_enrich.py | 55 +- misp_modules/modules/expansion/ods_enrich.py | 76 +- misp_modules/modules/expansion/odt_enrich.py | 61 +- .../modules/expansion/onion_lookup.py | 83 +- misp_modules/modules/expansion/onyphe.py | 301 ++-- misp_modules/modules/expansion/onyphe_full.py | 441 +++--- misp_modules/modules/expansion/otx.py | 103 +- misp_modules/modules/expansion/passive_ssh.py | 133 +- .../modules/expansion/passivetotal.py | 348 ++-- misp_modules/modules/expansion/pdf_enrich.py | 55 +- misp_modules/modules/expansion/pptx_enrich.py | 61 +- .../modules/expansion/qintel_qsentry.py | 198 +-- misp_modules/modules/expansion/qrcode.py | 87 +- .../modules/expansion/ransomcoindb.py | 78 +- misp_modules/modules/expansion/rbl.py | 68 +- .../modules/expansion/recordedfuture.py | 108 +- misp_modules/modules/expansion/reversedns.py | 72 +- .../modules/expansion/securitytrails.py | 506 +++--- misp_modules/modules/expansion/shodan.py | 247 ++- .../modules/expansion/sigma_queries.py | 68 +- .../expansion/sigma_syntax_validator.py | 55 +- .../modules/expansion/sigmf_expand.py | 217 +-- misp_modules/modules/expansion/socialscan.py | 95 +- .../modules/expansion/sophoslabs_intelix.py | 202 ++- misp_modules/modules/expansion/sourcecache.py | 60 +- misp_modules/modules/expansion/stairwell.py | 175 ++- .../stix2_pattern_syntax_validator.py | 52 +- misp_modules/modules/expansion/threatcrowd.py | 92 +- misp_modules/modules/expansion/threatfox.py | 63 +- misp_modules/modules/expansion/threatminer.py | 187 ++- .../modules/expansion/triage_submit.py | 93 +- .../modules/expansion/trustar_enrich.py | 160 +- misp_modules/modules/expansion/urlhaus.py | 169 +- misp_modules/modules/expansion/urlscan.py | 418 ++--- misp_modules/modules/expansion/variotdbs.py | 117 +- misp_modules/modules/expansion/virustotal.py | 312 ++-- .../modules/expansion/virustotal_public.py | 273 ++-- .../modules/expansion/virustotal_upload.py | 46 +- .../modules/expansion/vmray_submit.py | 109 +- misp_modules/modules/expansion/vmware_nsx.py | 73 +- misp_modules/modules/expansion/vulndb.py | 141 +- .../modules/expansion/vulnerability_lookup.py | 51 +- misp_modules/modules/expansion/vulners.py | 64 +- misp_modules/modules/expansion/vysion.py | 73 +- misp_modules/modules/expansion/whois.py | 68 +- misp_modules/modules/expansion/whoisfreaks.py | 225 +-- misp_modules/modules/expansion/wiki.py | 65 +- .../modules/expansion/xforceexchange.py | 194 ++- misp_modules/modules/expansion/xlsx_enrich.py | 63 +- misp_modules/modules/expansion/yara_query.py | 94 +- .../expansion/yara_syntax_validator.py | 49 +- misp_modules/modules/expansion/yeti.py | 205 +-- misp_modules/modules/export_mod/__init__.py | 3 - misp_modules/modules/export_mod/cef_export.py | 90 +- ...cisco_firesight_manager_ACL_rule_export.py | 108 +- .../export_mod/defender_endpoint_export.py | 88 +- .../modules/export_mod/goamlexport.py | 320 ++-- misp_modules/modules/export_mod/liteexport.py | 80 +- .../modules/export_mod/mass_eql_export.py | 40 +- .../modules/export_mod/nexthinkexport.py | 101 +- .../modules/export_mod/osqueryexport.py | 100 +- misp_modules/modules/export_mod/pdfexport.py | 87 +- misp_modules/modules/export_mod/testexport.py | 52 +- .../export_mod/threatStream_misp_export.py | 71 +- .../export_mod/threat_connect_export.py | 73 +- .../export_mod/virustotal_collections.py | 168 +- misp_modules/modules/export_mod/vt_graph.py | 137 +- .../modules/export_mod/yara_export.py | 237 +-- misp_modules/modules/import_mod/__init__.py | 22 - misp_modules/modules/import_mod/cof2misp.py | 164 +- misp_modules/modules/import_mod/csvimport.py | 294 +++- .../modules/import_mod/cuckooimport.py | 361 +++-- .../modules/import_mod/email_import.py | 211 ++- .../modules/import_mod/goamlimport.py | 392 +++-- .../modules/import_mod/import_blueprint.py | 68 +- misp_modules/modules/import_mod/joe_import.py | 48 +- .../modules/import_mod/lastline_import.py | 53 +- misp_modules/modules/import_mod/mispjson.py | 49 +- misp_modules/modules/import_mod/ocr.py | 70 +- .../modules/import_mod/openiocimport.py | 82 +- misp_modules/modules/import_mod/taxii21.py | 182 +-- misp_modules/modules/import_mod/testimport.py | 71 +- .../import_mod/threatanalyzer_import.py | 706 ++++++--- misp_modules/modules/import_mod/url_import.py | 84 +- .../modules/import_mod/vmray_import.py | 69 +- .../import_mod/vmray_summary_json_import.py | 39 +- poetry.lock | 1397 +++++++++++------ pyproject.toml | 63 +- tests/test.py | 631 ++++---- tests/test_actions.py | 12 +- tests/test_expansions.py | 589 +++---- tests/test_exports.py | 36 +- tests/test_imports.py | 15 +- tests/test_yara.py | 9 +- 209 files changed, 17304 insertions(+), 12000 deletions(-) create mode 100644 .pre-commit-config.yaml create mode 100755 documentation/generate.py delete mode 100755 documentation/generate_documentation.py create mode 100644 misp_modules/__main__.py create mode 100644 misp_modules/lib/ODTReader/LICENSE create mode 100644 misp_modules/lib/ODTReader/__init__.py create mode 100644 misp_modules/lib/ODTReader/odtreader.py create mode 100644 misp_modules/lib/ODTReader/win32_unicode_argv.py create mode 100644 misp_modules/lib/dnstrails/LICENSE create mode 100644 misp_modules/lib/dnstrails/__init__.py create mode 100644 misp_modules/lib/dnstrails/api.py create mode 100644 misp_modules/lib/dnstrails/exception.py create mode 100644 misp_modules/lib/onyphe/LICENSE create mode 100644 misp_modules/lib/onyphe/__init__.py create mode 100644 misp_modules/lib/onyphe/client.py create mode 100644 misp_modules/lib/onyphe/exception.py delete mode 100644 misp_modules/modules/expansion/google_search.py diff --git a/.github/workflows/release-package.yml b/.github/workflows/release-package.yml index 0704dfd8f..90e23e602 100644 --- a/.github/workflows/release-package.yml +++ b/.github/workflows/release-package.yml @@ -107,7 +107,7 @@ jobs: run: python -m pip install poetry - name: Install dependencies - run: poetry install --with unstable + run: poetry install -E all - name: Build package run: poetry build diff --git a/.github/workflows/test-package.yml b/.github/workflows/test-package.yml index cc9f68548..92475df3c 100644 --- a/.github/workflows/test-package.yml +++ b/.github/workflows/test-package.yml @@ -56,20 +56,16 @@ jobs: run: python -m pip install poetry - name: Install dependencies - run: poetry install --with test,unstable + run: poetry install --with test -E all - name: Build package run: poetry build - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - poetry run flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - poetry run flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + run: poetry run flake8 --extend-exclude=misp_modules/lib/,tests/,website/ - name: Run server in background - run: poetry run misp-modules -l 127.0.0.1 -s 2>error.log & + run: poetry run misp-modules -l 127.0.0.1 2>error.log & - name: Sleep for 10 seconds run: sleep 10s diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..dcaff6cb0 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,43 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: check-toml + - id: debug-statements + + - repo: local + hooks: + - id: black + name: black + entry: poetry run black + language: system + exclude: | + (?x)^( + website/.* + )$ + types_or: [python, pyi] + require_serial: true + + - id: isort + name: isort + entry: poetry run isort + language: system + exclude: | + (?x)^( + website/.* + )$ + types_or: [ python, pyi ] + require_serial: true + + - id: flake8 + name: flake8 + entry: poetry run flake8 + language: system + exclude: | + (?x)^( + misp_modules/lib/.*| + tests/.* + website/.*| + )$ + types_or: [ python, pyi ] + require_serial: true diff --git a/Makefile b/Makefile index e1a08f01b..84e1f5e6e 100644 --- a/Makefile +++ b/Makefile @@ -17,8 +17,8 @@ USE_DOCKER ?= prepare_docs: @echo "Preparing documentation." - poetry install --with docs,unstable - poetry run python $(DOCS_SRC_DIR)/generate_documentation.py + poetry install --with docs -E all + poetry run python $(DOCS_SRC_DIR)/generate.py mkdir -p $(DOCS_DIST_DIR)/logos mkdir -p $(DOCS_DIST_DIR)/img mkdir -p $(DOCS_DIST_DIR)/expansion/logos @@ -38,7 +38,7 @@ ifeq ($(USE_DOCKER), true) @echo "Generating documentation using '$(MKDOCS_DOCKER_IMAGE)'." docker run --rm -it -v $(PWD):/docs $(MKDOCS_DOCKER_IMAGE) build else - @echo "Generating docunentation." + @echo "Generating documentation." poetry run mkdocs build endif @@ -48,7 +48,7 @@ ifeq ($(USE_DOCKER), true) @echo "Deploying documentation using '$(MKDOCS_DOCKER_IMAGE)'." docker run --rm -it -v $(PWD):/docs -v /home/$(whoami)/.docker:/root/.docker:ro $(MKDOCS_DOCKER_IMAGE) gh-deploy else - @echo "Deploying docunentation." + @echo "Deploying documentation." poetry run mkdocs gh-deploy endif @@ -58,6 +58,6 @@ ifeq ($(USE_DOCKER), true) @echo "Serving documentation using '$(MKDOCS_DOCKER_IMAGE)'." docker run --rm -it -v $(PWD):/docs -p 8000:8000 $(MKDOCS_DOCKER_IMAGE) else - @echo "Serving docunentation." + @echo "Serving documentation." poetry run mkdocs serve endif diff --git a/README.md b/README.md index 10a0d10ab..a95b29881 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,6 @@ For further Information see the [license file](https://misp.github.io/misp-modul * [GeoIP City Lookup](https://misp.github.io/misp-modules/expansion/#geoip-city-lookup) - An expansion module to query a local copy of Maxmind's Geolite database with an IP address, in order to get information about the city where it is located. * [GeoIP Country Lookup](https://misp.github.io/misp-modules/expansion/#geoip-country-lookup) - Query a local copy of Maxminds Geolite database, updated for MMDB format * [Google Safe Browsing Lookup](https://misp.github.io/misp-modules/expansion/#google-safe-browsing-lookup) - Google safe browsing expansion module -* [Google Search](https://misp.github.io/misp-modules/expansion/#google-search) - An expansion hover module to expand google search information about an URL * [Google Threat Intelligence Lookup](https://misp.github.io/misp-modules/expansion/#google-threat-intelligence-lookup) - An expansion module to have the observable's threat score assessed by Google Threat Intelligence. * [GreyNoise Lookup](https://misp.github.io/misp-modules/expansion/#greynoise-lookup) - Module to query IP and CVE information from GreyNoise * [Hashdd Lookup](https://misp.github.io/misp-modules/expansion/#hashdd-lookup) - A hover module to check hashes against hashdd.com including NSLR dataset. @@ -159,6 +158,7 @@ For further Information see the [license file](https://misp.github.io/misp-modul * [Nexthink NXQL Export](https://misp.github.io/misp-modules/export_mod/#nexthink-nxql-export) - Nexthink NXQL query export module * [OSQuery Export](https://misp.github.io/misp-modules/export_mod/#osquery-export) - OSQuery export of a MISP event. * [Event to PDF Export](https://misp.github.io/misp-modules/export_mod/#event-to-pdf-export) - Simple export of a MISP event to PDF. +* [Test Export](https://misp.github.io/misp-modules/export_mod/#test-export) - Skeleton export module. * [ThreatStream Export](https://misp.github.io/misp-modules/export_mod/#threatstream-export) - Module to export a structured CSV file for uploading to threatStream. * [ThreadConnect Export](https://misp.github.io/misp-modules/export_mod/#threadconnect-export) - Module to export a structured CSV file for uploading to ThreatConnect. * [VirusTotal Collections Export](https://misp.github.io/misp-modules/export_mod/#virustotal-collections-export) - Creates a VT Collection from an event iocs. @@ -178,6 +178,7 @@ For further Information see the [license file](https://misp.github.io/misp-modul * [OCR Import](https://misp.github.io/misp-modules/import_mod/#ocr-import) - Optical Character Recognition (OCR) module for MISP. * [OpenIOC Import](https://misp.github.io/misp-modules/import_mod/#openioc-import) - Module to import OpenIOC packages. * [TAXII 2.1 Import](https://misp.github.io/misp-modules/import_mod/#taxii-2.1-import) - Import content from a TAXII 2.1 server +* [CSV Test Import](https://misp.github.io/misp-modules/import_mod/#csv-test-import) - Simple CSV import tool with mapable columns * [ThreadAnalyzer Sandbox Import](https://misp.github.io/misp-modules/import_mod/#threadanalyzer-sandbox-import) - Module to import ThreatAnalyzer archive.zip / analysis.json files. * [URL Import](https://misp.github.io/misp-modules/import_mod/#url-import) - Simple URL import tool with Faup * [VMRay API Import](https://misp.github.io/misp-modules/import_mod/#vmray-api-import) - Module to import VMRay (VTI) results. diff --git a/documentation/README.md b/documentation/README.md index 883009f5c..d95fe82e8 100644 --- a/documentation/README.md +++ b/documentation/README.md @@ -980,30 +980,6 @@ Google safe browsing expansion module ----- -#### [Google Search](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/google_search.py) - - - -An expansion hover module to expand google search information about an URL -[[source code](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/google_search.py)] - -- **features**: ->The module takes an url as input to query the Google search API. The result of the query is then return as raw text. - -- **input**: ->An url attribute. - -- **output**: ->Text containing the result of a Google search on the input url. - -- **references**: ->https://github.com/abenassi/Google-Search-API - -- **requirements**: ->The python Google Search API library - ------ - #### [Google Threat Intelligence Lookup](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/google_threat_intelligence.py) @@ -1272,6 +1248,9 @@ Module to query an IP ASN history service (https://github.com/D4-project/IPASN-H - **features**: >This module takes an IP address attribute as input and queries the CIRCL IPASN service. The result of the query is the latest asn related to the IP address, that is returned as a MISP object. +- **config**: +>custom_api + - **input**: >An IP address MISP attribute. @@ -3393,6 +3372,16 @@ Simple export of a MISP event to PDF. ----- +#### [Test Export](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/testexport.py) + +Skeleton export module. +[[source code](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/testexport.py)] + +- **features**: +> + +----- + #### [ThreatStream Export](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/threatStream_misp_export.py) @@ -3794,6 +3783,16 @@ Import content from a TAXII 2.1 server ----- +#### [CSV Test Import](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/testimport.py) + +Simple CSV import tool with mapable columns +[[source code](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/testimport.py)] + +- **features**: +> + +----- + #### [ThreadAnalyzer Sandbox Import](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/threatanalyzer_import.py) Module to import ThreatAnalyzer archive.zip / analysis.json files. diff --git a/documentation/generate.py b/documentation/generate.py new file mode 100755 index 000000000..11ea6e7a6 --- /dev/null +++ b/documentation/generate.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python +import collections +import copy +import importlib +import importlib.resources +import logging +import os +import pathlib +import sys + +logging.captureWarnings(True) + + +import misp_modules + +GH_LINK = "https://github.com/MISP/misp-modules/tree/main/misp_modules/modules" +GH_DOC_LINK = "https://misp.github.io/misp-modules" + +MODULE_TYPE_TITLE = { + misp_modules.ModuleType.EXPANSION.value: "Expansion Modules", + misp_modules.ModuleType.EXPORT_MOD.value: "Export Modules", + misp_modules.ModuleType.IMPORT_MOD.value: "Import Modules", + misp_modules.ModuleType.ACTION_MOD.value: "Action Modules", +} +MODULE_INFO_TO_IGNORE = ["module-type", "author", "version"] + +# ./ +BASE = pathlib.Path(__file__).resolve().parent.parent +# ./documentation/ +DOC_ROOT = pathlib.Path(__file__).resolve().parent +# ./misp_modules/ +SRC_ROOT = pathlib.Path(misp_modules.__file__).resolve().parent + +ALL_MODULE_INFO = collections.defaultdict(dict) + + +def _get_all_module_info() -> dict: + if not ALL_MODULE_INFO: + # Load libraries as root modules + misp_modules.promote_lib_to_root() + for module_type, module in misp_modules.iterate_modules(SRC_ROOT.joinpath(misp_modules.MODULES_DIR)): + module_name = os.path.splitext(module.name)[0] + module_package_name = ( + f"{misp_modules.__package__}.{misp_modules.MODULES_DIR}.{module_type.name}.{module_name}" + ) + try: + module = importlib.import_module(module_package_name) + module_info = copy.deepcopy(module.version()) + except ImportError: + continue # skip if we have issues loading the module + ALL_MODULE_INFO[module_type.name][module_name] = dict(sorted(module_info.items())) + + # sort for good measure + for module_type in list(ALL_MODULE_INFO.keys()): + ALL_MODULE_INFO[module_type] = dict(sorted(ALL_MODULE_INFO[module_type].items(), key=lambda item: item[0])) + return ALL_MODULE_INFO + + +def _generate_doc(module_type: str, logo_path: str = "logos") -> list[str]: + markdown = [] + gh_path = f"{GH_LINK}/{module_type}" + for module_name, module_info in _get_all_module_info()[module_type].items(): + gh_ref = f"{gh_path}/{module_name}.py" + module_info = copy.deepcopy(module_info) + for i in MODULE_INFO_TO_IGNORE: + module_info.pop(i) + try: + module_name_pretty = module_info.pop("name") + except KeyError: + exit(f"ERROR: Issue with module {module_name} - no field 'name' provided") + if module_name_pretty == "": + module_name_pretty = module_name + markdown.append(f"\n#### [{module_name_pretty}]({gh_ref})\n") + if module_info["logo"] != "": + logo = os.path.join(logo_path, module_info.pop("logo")) + markdown.append(f"\n\n") + if "description" in module_info: + markdown.append(f"\n{module_info.pop('description')}\n") + markdown.append(f"[[source code]({gh_ref})]\n") + if "features" in module_info: + markdown.append(_get_single_value("features", str(module_info.pop("features")).replace("\n", "\n>"))) + for field, value in sorted(module_info.items()): + if not value: + continue + if isinstance(value, list): + markdown.append(_handle_list(field, value)) + continue + markdown.append(_get_single_value(field, str(value).replace("\n", "\n>"))) + markdown.append("\n-----\n") + return markdown + + +def _generate_index_doc(module_type: str) -> list[str]: + markdown = [] + for module_name, module_info in _get_all_module_info()[module_type].items(): + module_name_pretty = module_info.get("name", module_name) + anchor_ref = f"{GH_DOC_LINK}/{module_type}/#{module_name_pretty.replace(' ', '-').lower()}" + description_without_newlines = module_info.get("description").replace("\n", " ") + markdown.append(f"* [{module_name_pretty}]({anchor_ref}) - {description_without_newlines}\n") + return markdown + + +def _get_single_value(field: str, value: str) -> str: + return f"\n- **{field}**:\n>{value}\n" + + +def _handle_list(field: str, values: list[str]) -> str: + if len(values) == 1: + return _get_single_value(field, values[0]) + values = "\n> - ".join(values) + return f"\n- **{field}**:\n> - {values}\n" + + +def write_doc_for_readme(): + markdown = ["# MISP modules documentation\n"] + for path, title in MODULE_TYPE_TITLE.items(): + markdown.append(f"\n## {title}\n") + markdown.extend(_generate_doc(path)) + with open(DOC_ROOT.joinpath("README.md"), "w") as w: + w.write("".join(markdown)) + + +def write_docs_for_mkdocs(): + for path, title in MODULE_TYPE_TITLE.items(): + markdown = _generate_doc(path, logo_path="../logos") + with open(os.path.join(DOC_ROOT.joinpath("mkdocs", f"{path}.md")), "w") as w: + w.write("".join(markdown)) + + +def update_docs_for_mkdocs_index(): + with open(DOC_ROOT.joinpath("mkdocs", "index.md"), "r") as r: + old_doc = r.readlines() + new_doc = [] + skip = False + for line in old_doc: + if skip and not line.startswith("## "): # find next title + continue # skip lines, as we're in the block that we're auto-generating + skip = False + new_doc.append(line) + if line.startswith("## Existing MISP modules"): + skip = True + for path, title in MODULE_TYPE_TITLE.items(): + new_doc.append(f"\n### {title}\n") + new_doc.extend(_generate_index_doc(path)) + new_doc.append("\n\n") + with open(DOC_ROOT.joinpath("mkdocs", "index.md"), "w") as w: + w.write("".join(new_doc)) + + +def update_readme(): + with open(BASE.joinpath("README.md"), "r") as r: + old_readme = r.readlines() + new_doc = [] + skip = False + for line in old_readme: + if skip and not line.startswith("# List of MISP modules"): # find next title + continue # skip lines, as we're in the block that we're auto-generating + new_doc.append(line) + if line.startswith("# List of MISP modules"): + skip = True + for path, title in MODULE_TYPE_TITLE.items(): + new_doc.append(f"\n## {title}\n") + new_doc.extend(_generate_index_doc(path)) + new_doc.append("\n\n") + with open(BASE.joinpath("README.md"), "w") as w: + w.write("".join(new_doc)) + + +def main(): + """Generate documentation.""" + write_doc_for_readme() + write_docs_for_mkdocs() + update_docs_for_mkdocs_index() + update_readme() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/documentation/generate_documentation.py b/documentation/generate_documentation.py deleted file mode 100755 index efab7ccad..000000000 --- a/documentation/generate_documentation.py +++ /dev/null @@ -1,178 +0,0 @@ -# -*- coding: utf-8 -*- -import os -from pathlib import Path -import importlib -import copy - -module_types = ['expansion', 'export_mod', 'import_mod', 'action_mod'] -titles = ['Expansion Modules', 'Export Modules', 'Import Modules', 'Action Modules'] -githublink = 'https://github.com/MISP/misp-modules/tree/main/misp_modules/modules' -githubiolink = 'https://misp.github.io/misp-modules' - -moduleinfo_to_ignore = ['module-type', 'author', 'version'] - -_all_moduleinfo = {} - - -def get_all_moduleinfo(): - ''' - Get all module information from the modules. - Behaves like a singleton, so it will only load the modules once. - ''' - if not _all_moduleinfo: - for module_type in module_types: - _all_moduleinfo[module_type] = {} - module_type_module = importlib.import_module(f"misp_modules.modules.{module_type}") - module_type_module.__all__.sort() - for module_name in module_type_module.__all__: - module_package_name = f"misp_modules.modules.{module_type}.{module_name}" - try: - module = importlib.import_module(module_package_name) - moduleinfo = copy.deepcopy(module.version()) - except Exception: - continue # skip if we have issues loading the module - - moduleinfo = dict(sorted(moduleinfo.items())) - _all_moduleinfo[module_type][module_name] = moduleinfo - - return _all_moduleinfo - - -def generate_doc(module_type, root_path, logo_path='logos'): - markdown = [] - # current_path = os.path.join(root_path, 'website', module_type) - # files = sorted(os.listdir(current_path)) - githubpath = f'{githublink}/{module_type}' - - for module_name, moduleinfo in get_all_moduleinfo()[module_type].items(): - githubref = f'{githubpath}/{module_name}.py' - - moduleinfo = copy.deepcopy(moduleinfo) # ensure to not modify the original data - for i in moduleinfo_to_ignore: - moduleinfo.pop(i) - - try: - module_name_pretty = moduleinfo.pop('name') - except KeyError: - exit(f"ERROR: Issue with module {module_name} - no field 'name' provided") - if module_name_pretty == '': - module_name_pretty = module_name - - markdown.append(f'\n#### [{module_name_pretty}]({githubref})\n') - if moduleinfo['logo'] != '': - logo = os.path.join(logo_path, moduleinfo.pop('logo')) - markdown.append(f"\n\n") - if 'description' in moduleinfo: - markdown.append(f"\n{moduleinfo.pop('description')}\n") - markdown.append(f"[[source code]({githubref})]\n") - if 'features' in moduleinfo: - markdown.append(get_single_value('features', str(moduleinfo.pop('features')).replace('\n', '\n>'))) - for field, value in sorted(moduleinfo.items()): - if not value: - continue - if isinstance(value, list): - markdown.append(handle_list(field, value)) - continue - markdown.append(get_single_value(field, str(value).replace('\n', '\n>'))) - markdown.append('\n-----\n') - return markdown - - -def generate_index_doc(module_type, root_path): - markdown = [] - for module_name, moduleinfo in get_all_moduleinfo()[module_type].items(): - module_name_pretty = moduleinfo.get('name') - if module_name_pretty == '': - module_name_pretty = module_name - - anchor_ref = f"{githubiolink}/{module_type}/#{module_name_pretty.replace(' ', '-').lower()}" - description_without_newlines = moduleinfo.get("description").replace('\n', ' ') - markdown.append(f'* [{module_name_pretty}]({anchor_ref}) - {description_without_newlines}\n') - return markdown - - -def get_single_value(field, value): - return f"\n- **{field}**:\n>{value}\n" - - -def handle_list(field, values): - if len(values) == 1: - return get_single_value(field, values[0]) - values = '\n> - '.join(values) - return f"\n- **{field}**:\n> - {values}\n" - - -def write_doc_for_readme(root_path): - markdown = ["# MISP modules documentation\n"] - for _path, title in zip(module_types, titles): - markdown.append(f'\n## {title}\n') - markdown.extend(generate_doc(_path, root_path)) - with open(root_path / 'README.md', 'w') as w: - w.write(''.join(markdown)) - - -def write_docs_for_mkdocs(root_path): - for _path, title in zip(module_types, titles): - markdown = generate_doc(_path, root_path, logo_path='../logos') - with open(os.path.join(root_path, 'mkdocs', f'{_path}.md'), 'w') as w: - w.write(''.join(markdown)) - - -def update_docs_for_mkdocs_index(root_path): - with open(root_path / 'mkdocs' / 'index.md', 'r') as r: - old_doc = r.readlines() - - new_doc = [] - skip = False - for line in old_doc: - if skip and not line.startswith('## '): # find next title - continue # skip lines, as we're in the block that we're auto-generating - - skip = False - new_doc.append(line) - - if line.startswith('## Existing MISP modules'): - skip = True - # generate the updated content - for _path, title in zip(module_types, titles): - new_doc.append(f'\n### {title}\n') - new_doc.extend(generate_index_doc(_path, root_path)) - new_doc.append('\n\n') - - with open(root_path / 'mkdocs' / 'index.md', 'w') as w: - w.write(''.join(new_doc)) - pass - - -def update_readme(root_path): - with open(root_path / 'README.md', 'r') as r: - old_readme = r.readlines() - - new_doc = [] - skip = False - for line in old_readme: - if skip and not line.startswith('# List of MISP modules'): # find next title - continue # skip lines, as we're in the block that we're auto-generating - - new_doc.append(line) - - if line.startswith('# List of MISP modules'): - skip = True - # generate the updated content - for _path, title in zip(module_types, titles): - new_doc.append(f'\n## {title}\n') - new_doc.extend(generate_index_doc(_path, root_path)) - new_doc.append('\n\n') - - with open(root_path / 'README.md', 'w') as w: - w.write(''.join(new_doc)) - pass - - -if __name__ == '__main__': - root_path = Path(__file__).resolve().parent - - write_doc_for_readme(root_path) - write_docs_for_mkdocs(root_path) - update_docs_for_mkdocs_index(root_path) - update_readme(root_path.parent) diff --git a/documentation/mkdocs/expansion.md b/documentation/mkdocs/expansion.md index 7a1cf9c41..eb19e5712 100644 --- a/documentation/mkdocs/expansion.md +++ b/documentation/mkdocs/expansion.md @@ -977,30 +977,6 @@ Google safe browsing expansion module ----- -#### [Google Search](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/google_search.py) - - - -An expansion hover module to expand google search information about an URL -[[source code](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/google_search.py)] - -- **features**: ->The module takes an url as input to query the Google search API. The result of the query is then return as raw text. - -- **input**: ->An url attribute. - -- **output**: ->Text containing the result of a Google search on the input url. - -- **references**: ->https://github.com/abenassi/Google-Search-API - -- **requirements**: ->The python Google Search API library - ------ - #### [Google Threat Intelligence Lookup](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/google_threat_intelligence.py) @@ -1269,6 +1245,9 @@ Module to query an IP ASN history service (https://github.com/D4-project/IPASN-H - **features**: >This module takes an IP address attribute as input and queries the CIRCL IPASN service. The result of the query is the latest asn related to the IP address, that is returned as a MISP object. +- **config**: +>custom_api + - **input**: >An IP address MISP attribute. diff --git a/documentation/mkdocs/export_mod.md b/documentation/mkdocs/export_mod.md index bd369f110..de497948d 100644 --- a/documentation/mkdocs/export_mod.md +++ b/documentation/mkdocs/export_mod.md @@ -248,6 +248,16 @@ Simple export of a MISP event to PDF. ----- +#### [Test Export](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/testexport.py) + +Skeleton export module. +[[source code](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/testexport.py)] + +- **features**: +> + +----- + #### [ThreatStream Export](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/threatStream_misp_export.py) diff --git a/documentation/mkdocs/import_mod.md b/documentation/mkdocs/import_mod.md index ca6ab72a2..cc1517634 100644 --- a/documentation/mkdocs/import_mod.md +++ b/documentation/mkdocs/import_mod.md @@ -251,6 +251,16 @@ Import content from a TAXII 2.1 server ----- +#### [CSV Test Import](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/testimport.py) + +Simple CSV import tool with mapable columns +[[source code](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/testimport.py)] + +- **features**: +> + +----- + #### [ThreadAnalyzer Sandbox Import](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/threatanalyzer_import.py) Module to import ThreatAnalyzer archive.zip / analysis.json files. diff --git a/documentation/mkdocs/index.md b/documentation/mkdocs/index.md index 171740b02..4885d24d7 100644 --- a/documentation/mkdocs/index.md +++ b/documentation/mkdocs/index.md @@ -49,7 +49,6 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj * [GeoIP City Lookup](https://misp.github.io/misp-modules/expansion/#geoip-city-lookup) - An expansion module to query a local copy of Maxmind's Geolite database with an IP address, in order to get information about the city where it is located. * [GeoIP Country Lookup](https://misp.github.io/misp-modules/expansion/#geoip-country-lookup) - Query a local copy of Maxminds Geolite database, updated for MMDB format * [Google Safe Browsing Lookup](https://misp.github.io/misp-modules/expansion/#google-safe-browsing-lookup) - Google safe browsing expansion module -* [Google Search](https://misp.github.io/misp-modules/expansion/#google-search) - An expansion hover module to expand google search information about an URL * [Google Threat Intelligence Lookup](https://misp.github.io/misp-modules/expansion/#google-threat-intelligence-lookup) - An expansion module to have the observable's threat score assessed by Google Threat Intelligence. * [GreyNoise Lookup](https://misp.github.io/misp-modules/expansion/#greynoise-lookup) - Module to query IP and CVE information from GreyNoise * [Hashdd Lookup](https://misp.github.io/misp-modules/expansion/#hashdd-lookup) - A hover module to check hashes against hashdd.com including NSLR dataset. @@ -138,6 +137,7 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj * [Nexthink NXQL Export](https://misp.github.io/misp-modules/export_mod/#nexthink-nxql-export) - Nexthink NXQL query export module * [OSQuery Export](https://misp.github.io/misp-modules/export_mod/#osquery-export) - OSQuery export of a MISP event. * [Event to PDF Export](https://misp.github.io/misp-modules/export_mod/#event-to-pdf-export) - Simple export of a MISP event to PDF. +* [Test Export](https://misp.github.io/misp-modules/export_mod/#test-export) - Skeleton export module. * [ThreatStream Export](https://misp.github.io/misp-modules/export_mod/#threatstream-export) - Module to export a structured CSV file for uploading to threatStream. * [ThreadConnect Export](https://misp.github.io/misp-modules/export_mod/#threadconnect-export) - Module to export a structured CSV file for uploading to ThreatConnect. * [VirusTotal Collections Export](https://misp.github.io/misp-modules/export_mod/#virustotal-collections-export) - Creates a VT Collection from an event iocs. @@ -157,6 +157,7 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj * [OCR Import](https://misp.github.io/misp-modules/import_mod/#ocr-import) - Optical Character Recognition (OCR) module for MISP. * [OpenIOC Import](https://misp.github.io/misp-modules/import_mod/#openioc-import) - Module to import OpenIOC packages. * [TAXII 2.1 Import](https://misp.github.io/misp-modules/import_mod/#taxii-2.1-import) - Import content from a TAXII 2.1 server +* [CSV Test Import](https://misp.github.io/misp-modules/import_mod/#csv-test-import) - Simple CSV import tool with mapable columns * [ThreadAnalyzer Sandbox Import](https://misp.github.io/misp-modules/import_mod/#threadanalyzer-sandbox-import) - Module to import ThreatAnalyzer archive.zip / analysis.json files. * [URL Import](https://misp.github.io/misp-modules/import_mod/#url-import) - Simple URL import tool with Faup * [VMRay API Import](https://misp.github.io/misp-modules/import_mod/#vmray-api-import) - Module to import VMRay (VTI) results. diff --git a/misp_modules/__init__.py b/misp_modules/__init__.py index 3848d8d22..580248ce3 100644 --- a/misp_modules/__init__.py +++ b/misp_modules/__init__.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Core MISP expansion modules loader and web service # @@ -17,322 +16,95 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . - -import os -import signal -import sys +import enum import importlib -import logging -import fnmatch -import argparse -import re -import datetime -import psutil -import pkgutil -import platform +import importlib.abc +import importlib.resources +import importlib.util +import pathlib +import sys +import types import typing -try: - import orjson as json -except ImportError: - import json - -import tornado.web -import tornado.process -from tornado.ioloop import IOLoop -from tornado.concurrent import run_on_executor -from concurrent.futures import ThreadPoolExecutor -from pymisp import pymisp_json_default - - -import warnings -warnings.filterwarnings("ignore", category=SyntaxWarning) - - -try: - from .modules import * # noqa - HAS_PACKAGE_MODULES = True -except Exception as e: - logging.exception(e) - HAS_PACKAGE_MODULES = False - -try: - from .helpers import * # noqa - HAS_PACKAGE_HELPERS = True -except Exception as e: - logging.exception(e) - HAS_PACKAGE_HELPERS = False - -log = logging.getLogger('misp-modules') - - -def handle_signal(sig, frame): - IOLoop.instance().add_callback_from_signal(IOLoop.instance().stop) - - -def init_logger(debug=False): - formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') - handler = logging.StreamHandler() - handler.setFormatter(formatter) - - # Enable access logs - access_log = logging.getLogger('tornado.access') - access_log.propagate = False - access_log.setLevel(logging.INFO) - access_log.addHandler(handler) - - # Set application log - log.addHandler(handler) - log.propagate = False - log.setLevel(logging.DEBUG if debug else logging.INFO) - - -def load_helpers(helpersdir): - sys.path.append(helpersdir) - hhandlers = {} - helpers = [] - for root, dirnames, filenames in os.walk(helpersdir): - if os.path.basename(root) == '__pycache__': - continue - if re.match(r'^\.', os.path.basename(root)): - continue - for filename in fnmatch.filter(filenames, '*.py'): - if filename == '__init__.py': - continue - helpername = filename.split(".")[0] - hhandlers[helpername] = importlib.import_module(helpername) - selftest = hhandlers[helpername].selftest() - if selftest is None: - helpers.append(helpername) - log.info(f'Helpers loaded {filename}') - else: - log.warning(f'Helpers failed {filename} due to {selftest}') - - -def load_package_helpers(): - if not HAS_PACKAGE_HELPERS: - log.error('Unable to load MISP helpers from package.') - sys.exit(1) - mhandlers = {} - helpers = [] - for path, helper in sys.modules.items(): - if not path.startswith('misp_modules.helpers.'): - continue - helper_name = path.replace('misp_modules.helpers.', '') - mhandlers[helper_name] = helper - selftest = mhandlers[helper_name].selftest() - if selftest is None: - helpers.append(helper_name) - log.info(f'Helper loaded {helper_name}') - else: - log.warning(f'Helpers failed {helper_name} due to {selftest}') - return mhandlers, helpers - - -def load_modules(mod_dir): - sys.path.append(mod_dir) - mhandlers = {} - modules = [] - for root, dirnames, filenames in os.walk(mod_dir): - if os.path.basename(root) == '__pycache__': - continue - if os.path.basename(root).startswith("."): - continue - for filename in fnmatch.filter(filenames, '*.py'): - if root.split('/')[-1].startswith('_'): - continue - if filename == '__init__.py': - continue - module_name = filename.split(".")[0] - module_type = os.path.split(mod_dir)[1] - try: - mhandlers[module_name] = importlib.import_module(os.path.basename(root) + '.' + module_name) - except Exception as e: - log.warning(f'MISP modules {module_name} failed due to {e}') - continue - modules.append(module_name) - log.info(f'MISP modules {module_name} imported') - mhandlers['type:' + module_name] = module_type - return mhandlers, modules - - -def load_package_modules(): - if not HAS_PACKAGE_MODULES: - log.error('Unable to load MISP modules from package.') - sys.exit(1) - mhandlers = {} - modules = [] - for path, module in sys.modules.items(): - r = re.findall(r"misp_modules[.]modules[.](\w+)[.]([^_]\w+)", path) - if r and len(r[0]) == 2: - module_type, module_name = r[0] - mhandlers[module_name] = module - modules.append(module_name) - log.info(f'MISP modules {module_name} imported') - mhandlers['type:' + module_name] = module_type - return mhandlers, modules - - -class Healthcheck(tornado.web.RequestHandler): - def get(self): - self.write(b'{"status": true}') - - -class ListModules(tornado.web.RequestHandler): - global loaded_modules - global mhandlers - - _cached_json = None - - def get(self): - if not self._cached_json: - ret = [] - for module_name in loaded_modules: - ret.append({ - 'name': module_name, - 'type': mhandlers['type:' + module_name], - 'mispattributes': mhandlers[module_name].introspection(), - 'meta': mhandlers[module_name].version() - }) - self._cached_json = json.dumps(ret) - - log.debug('MISP ListModules request') - self.write(self._cached_json) - - -class QueryModule(tornado.web.RequestHandler): - - # Default value in Python 3.5 - # https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor - nb_threads = tornado.process.cpu_count() * 5 - executor = ThreadPoolExecutor(nb_threads) - - @run_on_executor - def run_request(self, module_name, json_payload, dict_payload): - log.debug('MISP QueryModule %s request %s', module_name, json_payload) - module = mhandlers[module_name] - if getattr(module, "dict_handler", None): - # New method that avoids double JSON decoding, new modules should define dict_handler - response = module.dict_handler(request=dict_payload) - else: - response = module.handler(q=json_payload) - return json.dumps(response, default=pymisp_json_default) - - @tornado.gen.coroutine - def post(self): - try: - json_payload = self.request.body - dict_payload = json.loads(json_payload) - if dict_payload.get('timeout'): - timeout = datetime.timedelta(seconds=int(dict_payload.get('timeout'))) - else: - timeout = datetime.timedelta(seconds=300) - future = self.run_request(dict_payload['module'], json_payload, dict_payload) - response = yield tornado.gen.with_timeout(timeout, future) - self.write(response) - except tornado.gen.TimeoutError: - log.warning('Timeout on {}'.format(dict_payload['module'])) - self.write(json.dumps({'error': 'Timeout.'})) - except Exception: - self.write(json.dumps({'error': 'Something went wrong, look in the server logs for details'})) - log.exception('Something went wrong when processing query request') - finally: - self.finish() - +import psutil -def _launch_from_current_dir(): - log.info('Launch MISP modules server from current directory.') - os.chdir(os.path.dirname(__file__)) - modulesdir = 'modules' - helpersdir = 'helpers' - load_helpers(helpersdir=helpersdir) - return load_modules(modulesdir) +# Constants +LIBRARY_DIR = "lib" +MODULES_DIR = "modules" +HELPERS_DIR = "helpers" -def main(): - global mhandlers - global loaded_modules - signal.signal(signal.SIGINT, handle_signal) - signal.signal(signal.SIGTERM, handle_signal) +class ModuleType(enum.Enum): + """All the modules types.""" - arg_parser = argparse.ArgumentParser(description='misp-modules server', formatter_class=argparse.RawTextHelpFormatter) - arg_parser.add_argument('-t', '--test', default=False, action='store_true', help='Test mode') - arg_parser.add_argument('-s', '--system', default=False, action='store_true', help='Run a system install (package installed via pip)') - arg_parser.add_argument('-d', '--debug', default=False, action='store_true', help='Enable debugging') - arg_parser.add_argument('-p', '--port', default=6666, help='misp-modules TCP port (default 6666)') - arg_parser.add_argument('-l', '--listen', default='localhost', help='misp-modules listen address (default localhost)') - arg_parser.add_argument('-m', default=[], action='append', help='Register a custom module') - arg_parser.add_argument('--devel', default=False, action='store_true', help='''Start in development mode, enable debug, start only the module(s) listed in -m.\nExample: -m misp_modules.modules.expansion.bgpranking''') - args = arg_parser.parse_args() + EXPANSION = "expansion" + EXPORT_MOD = "export_mod" + IMPORT_MOD = "import_mod" + ACTION_MOD = "action_mod" - if args.devel: - init_logger(debug=True) - log.info('Launch MISP modules server in development mode. Enable debug, load a list of modules is -m is used.') - if args.m: - mhandlers = {} - modules = [] - for module in args.m: - splitted = module.split(".") - modulename = splitted[-1] - moduletype = splitted[2] - mhandlers[modulename] = importlib.import_module(module) - mhandlers['type:' + modulename] = moduletype - modules.append(modulename) - log.info(f'MISP modules {modulename} imported') - else: - mhandlers, loaded_modules = _launch_from_current_dir() - else: - init_logger(debug=args.debug) - if args.system: - log.info('Launch MISP modules server from package.') - load_package_helpers() - mhandlers, loaded_modules = load_package_modules() - else: - mhandlers, loaded_modules = _launch_from_current_dir() - for module in args.m: - mispmod = importlib.import_module(module) - mispmod.register(mhandlers, loaded_modules) +def is_valid_module(module: importlib.abc.Traversable) -> bool: + """Whether the reference is a valid module file.""" + if not module.is_file(): + return False + if module.name == "__init__.py": + return False + if not module.name.endswith(".py"): + return False + return True - service = [ - (r'/modules', ListModules), - (r'/query', QueryModule), - (r'/healthcheck', Healthcheck), - ] - application = tornado.web.Application(service) +def is_valid_module_type(module_type: importlib.abc.Traversable) -> bool: + """Whether the reference is a valid module type.""" + if not module_type.is_dir(): + return False try: - server = tornado.httpserver.HTTPServer(application, max_buffer_size=1073741824) # buffer size increase when large MISP event are submitted - GH issue 662 - server.listen(args.port, args.listen) - except Exception as e: - if e.errno == 98: - pids = psutil.pids() - for pid in pids: - p = psutil.Process(pid) - if p.name() == "misp-modules": - print("\n\n\n") - print(e) - print("\nmisp-modules is still running as PID: {}\n".format(pid)) - print("Please kill accordingly:") - print("sudo kill {}".format(pid)) - return 1 - print(e) - print("misp-modules might still be running.") - else: - log.exception(f"Could not listen on {args.listen}:{args.port}") - return 1 - - log.info(f'MISP modules server started on {args.listen} port {args.port}') - if args.test: - log.info('MISP modules started in test-mode, quitting immediately.') - return 0 + ModuleType(module_type.name) + except ValueError: + return False + return True + + +def iterate_helpers( + helpers_dir: typing.Union[importlib.abc.Traversable, pathlib.Path], +) -> typing.Generator[importlib.abc.Traversable, None, None]: + """Iterate helpers and return helper references.""" + for helper in helpers_dir.iterdir(): + if is_valid_module(helper): + yield helper + + +def iterate_modules( + modules_dir: typing.Union[importlib.abc.Traversable, pathlib.Path], +) -> typing.Generator[tuple[importlib.abc.Traversable, importlib.abc.Traversable], None, None]: + """Iterate modules and return both module types and module references.""" + for module_type in modules_dir.iterdir(): + if is_valid_module_type(module_type): + for module in module_type.iterdir(): + if is_valid_module(module): + yield module_type, module + + +def import_from_path(module_name: str, file_path: str) -> types.ModuleType: + """Import module from any point in the file system.""" + spec = importlib.util.spec_from_file_location(module_name, file_path) + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + return module + + +def get_misp_modules_pid() -> typing.Union[int, None]: + """Get the pid of any process that have `misp-modules` in the command line.""" try: - IOLoop.instance().start() - finally: - IOLoop.instance().stop() - - return 0 + for pid in psutil.pids(): + if any("misp-modules" in x for x in psutil.Process(pid).cmdline()): + return pid + return None + except psutil.AccessDenied: + return None -if __name__ == '__main__': - sys.exit(main()) +def promote_lib_to_root() -> None: + """Nested libraries are called as full fledge libraries.""" + sys.path.append(str(importlib.resources.files(__package__).joinpath(LIBRARY_DIR))) diff --git a/misp_modules/__main__.py b/misp_modules/__main__.py new file mode 100644 index 000000000..09d9ec8e9 --- /dev/null +++ b/misp_modules/__main__.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python +# +# Core MISP expansion modules loader and web service +# +# Copyright (C) 2016 Alexandre Dulaunoy +# Copyright (C) 2016 CIRCL - Computer Incident Response Center Luxembourg +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +import importlib +import logging +import os.path +import pathlib +import signal +import sys + +logging.captureWarnings(True) + +import argparse +import datetime +import importlib.resources +import importlib.util +import types +from concurrent.futures import ThreadPoolExecutor + +import orjson +import pymisp +import tornado.process +import tornado.web +from tornado import concurrent as tornado_concurrent +from tornado import ioloop + +import misp_modules + +# See https://github.com/MISP/misp-modules/issues/662 +MAX_BUFFER_SIZE = 1073741824 + +# Global variables +MODULES_HANDLERS = {} +HELPERS_HANDLERS = {} +LOGGER = logging.getLogger("misp-modules") + + +def handle_signal(sig: int, frame: types.FrameType) -> None: + """Handle the signal.""" + _ = sig, frame + ioloop.IOLoop.instance().add_callback_from_signal(ioloop.IOLoop.instance().stop) + + +def init_logger(debug: bool = False) -> None: + """Initialize the logger.""" + formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") + handler = logging.StreamHandler() + handler.setFormatter(formatter) + + # Enable access logs + access_log = logging.getLogger("tornado.access") + access_log.propagate = False + access_log.setLevel(logging.INFO) + access_log.addHandler(handler) + + # Configure warning logs + warning_log = logging.getLogger("py.warnings") + warning_log.propagate = False + warning_log.setLevel(logging.ERROR) + warning_log.addHandler(handler) + + # Set application log + LOGGER.propagate = False + LOGGER.setLevel(logging.DEBUG if debug else logging.INFO) + LOGGER.addHandler(handler) + + +class Healthcheck(tornado.web.RequestHandler): + """Healthcheck handler.""" + + def get(self): + LOGGER.debug("MISP Healthcheck request") + self.write(b'{"status": true}') + + +class ListModules(tornado.web.RequestHandler): + """ListModules handler.""" + + CACHE = None + + @classmethod + def _build_handlers_data(cls) -> bytes: + return orjson.dumps( + [ + { + "name": module_name, + "type": MODULES_HANDLERS["type:" + module_name], + "mispattributes": MODULES_HANDLERS[module_name].introspection(), + "meta": MODULES_HANDLERS[module_name].version(), + } + for module_name in MODULES_HANDLERS + if not module_name.startswith("type:") + ] + ) + + def get(self): + LOGGER.debug("MISP ListModules request") + if not self.CACHE: + self.CACHE = self._build_handlers_data() + self.write(self.CACHE) + + +class QueryModule(tornado.web.RequestHandler): + """QueryModule handler.""" + + DEFAULT_TIMEOUT = 300 + + # Never go above 32 + executor = ThreadPoolExecutor(max_workers=min(32, tornado.process.cpu_count() * 5)) + + @tornado_concurrent.run_on_executor + def run_request(self, module_name, json_payload, dict_payload): + LOGGER.debug("MISP QueryModule %s request %s", module_name, json_payload) + try: + response = MODULES_HANDLERS[module_name].dict_handler(request=dict_payload) + except AttributeError: + response = MODULES_HANDLERS[module_name].handler(q=json_payload) + return orjson.dumps(response, default=pymisp.pymisp_json_default) + + @tornado.gen.coroutine + def post(self): + json_payload = self.request.body + dict_payload = orjson.loads(json_payload) + timeout = datetime.timedelta(seconds=int(dict_payload.get("timeout", self.DEFAULT_TIMEOUT))) + try: + future = self.run_request(dict_payload["module"], json_payload, dict_payload) + response = yield tornado.gen.with_timeout(timeout, future) + self.write(response) + except tornado.gen.TimeoutError: + LOGGER.warning("Timeout on {}".format(dict_payload["module"])) + self.write(orjson.dumps({"error": "Timeout."})) + except Exception: + self.write(orjson.dumps({"error": "Something went wrong, look in the server logs for details"})) + LOGGER.exception("Something went wrong when processing query request") + finally: + self.finish() + + +def main(): + """Init function.""" + global HELPERS_HANDLERS + global MODULES_HANDLERS + + signal.signal(signal.SIGINT, handle_signal) + signal.signal(signal.SIGTERM, handle_signal) + + arg_parser = argparse.ArgumentParser(description="misp-modules", formatter_class=argparse.RawTextHelpFormatter) + arg_parser.add_argument("-t", "--test", default=False, action="store_true", help="test mode") + arg_parser.add_argument("-d", "--debug", default=False, action="store_true", help="enable debugging") + arg_parser.add_argument("-p", "--port", type=int, default=6666, help="port (default 6666)") + arg_parser.add_argument("-l", "--listen", default="localhost", help="address (default localhost)") + arg_parser.add_argument("-c", "--custom", default=None, help="custom modules root") + arg_parser.add_argument("-s", "--system", default=None, help="legacy option that now has no effect") + args = arg_parser.parse_args() + + # Initialize + init_logger(debug=args.debug) + + # Load libraries as root modules + misp_modules.promote_lib_to_root() + + # Load helpers + for helper in misp_modules.iterate_helpers( + importlib.resources.files(__package__).joinpath(misp_modules.HELPERS_DIR) + ): + helper_name = os.path.splitext(helper.name)[0] + absolute_helper_name = ".".join([__package__, misp_modules.HELPERS_DIR, helper_name]) + try: + imported_helper = importlib.import_module(absolute_helper_name) + if test_error := imported_helper.selftest(): + raise RuntimeError(test_error) + except (ImportError, RuntimeError) as e: + LOGGER.warning("Helper %s failed: %s", helper_name, e) + continue + HELPERS_HANDLERS[helper_name] = imported_helper + LOGGER.info("Helper %s loaded", helper_name) + + # Load modules + for module_type, module in misp_modules.iterate_modules( + importlib.resources.files(__package__).joinpath(misp_modules.MODULES_DIR) + ): + module_name = os.path.splitext(module.name)[0] + absolute_module_name = ".".join([__package__, misp_modules.MODULES_DIR, module_type.name, module_name]) + try: + imported_module = importlib.import_module(absolute_module_name) + except ImportError as e: + LOGGER.warning("MISP module %s (type=%s) failed: %s", module_name, module_type.name, e) + continue + MODULES_HANDLERS[module_name] = imported_module + MODULES_HANDLERS[f"type:{module_name}"] = module_type.name + LOGGER.info("MISP module %s (type=%s) imported", module_name, module_type.name) + + # Load custom modules + if args.custom: + LOGGER.info("Parsing custom modules from root directory: %s", args.custom) + for module_type, module in misp_modules.iterate_modules(pathlib.Path(args.custom)): + module_name = os.path.splitext(module.name)[0] + try: + imported_module = misp_modules.import_from_path(module_name, str(module_type.joinpath(module.name))) + except ImportError as e: + LOGGER.warning("CUSTOM MISP module %s (type=%s) failed: %s", module_name, module_type.name, e) + continue + MODULES_HANDLERS[module_name] = imported_module + MODULES_HANDLERS[f"type:{module_name}"] = module_type.name + LOGGER.info("CUSTOM MISP module %s (type=%s) imported", module_name, module_type.name) + + try: + server = tornado.httpserver.HTTPServer( + tornado.web.Application( + [ + (r"/modules", ListModules), + (r"/query", QueryModule), + (r"/healthcheck", Healthcheck), + ] + ), + max_buffer_size=MAX_BUFFER_SIZE, + ) + server.listen(args.port, args.listen) + except OSError as e: + if e.errno == 48 or e.errno == 98: + LOGGER.exception("Could not listen on %s:%d", args.listen, args.port) + if pid := misp_modules.get_misp_modules_pid(): + LOGGER.exception("Dangling 'misp-modules' with pid %d found", pid) + else: + LOGGER.exception("Unspecified OSError") + raise + except Exception: + LOGGER.exception("Unspecified Exception") + raise + + LOGGER.info("MISP modules server started on %s:%d", args.listen, args.port) + if args.test: + LOGGER.info("MISP modules started in test-mode, quitting immediately.") + return 0 + + try: + ioloop.IOLoop.instance().start() + finally: + ioloop.IOLoop.instance().stop() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/misp_modules/helpers/__init__.py b/misp_modules/helpers/__init__.py index cd16c8804..e69de29bb 100644 --- a/misp_modules/helpers/__init__.py +++ b/misp_modules/helpers/__init__.py @@ -1 +0,0 @@ -__all__ = ['cache'] diff --git a/misp_modules/helpers/cache.py b/misp_modules/helpers/cache.py index 7b9821325..aa9d0a990 100644 --- a/misp_modules/helpers/cache.py +++ b/misp_modules/helpers/cache.py @@ -19,33 +19,36 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import hashlib import os +import typing import redis -import hashlib port = int(os.getenv("REDIS_PORT")) if os.getenv("REDIS_PORT") else 6379 -hostname = os.getenv("REDIS_BACKEND") or '127.0.0.1' +hostname = os.getenv("REDIS_BACKEND") or "127.0.0.1" db = int(os.getenv("REDIS_DATABASE")) if os.getenv("REDIS_DATABASE") else 0 password = os.getenv("REDIS_PW") or None -def selftest(enable=True): +def selftest(enable: bool = True) -> typing.Union[str, None]: if not enable: - return False + return None r = redis.Redis(host=hostname, password=password, port=port, db=db) try: r.ping() except Exception: - return 'Redis not running or not installed. Helper will be disabled.' + return "Redis not running or not installed. Helper will be disabled." + else: + return None def get(modulename=None, query=None, value=None, debug=False): - if (modulename is None or query is None): + if modulename is None or query is None: return False r = redis.Redis(host=hostname, password=password, port=port, db=db, decode_responses=True) h = hashlib.sha1() - h.update(query.encode('UTF-8')) + h.update(query.encode("UTF-8")) hv = h.hexdigest() key = "m:{}:{}".format(modulename, hv) @@ -68,17 +71,18 @@ def flush(): if __name__ == "__main__": import sys + if selftest() is not None: sys.exit() else: print("Selftest ok") v = get(modulename="testmodule", query="abcdef", value="barfoo", debug=True) - if v == 'barfoo': + if v == "barfoo": print("Cache ok") v = get(modulename="testmodule", query="abcdef") print(v) v = get(modulename="testmodule") - if (not v): + if not v: print("Failed ok") if flush(): print("Cache flushed ok") diff --git a/misp_modules/lib/ODTReader/LICENSE b/misp_modules/lib/ODTReader/LICENSE new file mode 100644 index 000000000..9cecc1d46 --- /dev/null +++ b/misp_modules/lib/ODTReader/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + {one line to give the program's name and a brief idea of what it does.} + Copyright (C) {year} {name of author} + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + {project} Copyright (C) {year} {fullname} + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/misp_modules/lib/ODTReader/__init__.py b/misp_modules/lib/ODTReader/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/misp_modules/lib/ODTReader/odtreader.py b/misp_modules/lib/ODTReader/odtreader.py new file mode 100644 index 000000000..ca755158d --- /dev/null +++ b/misp_modules/lib/ODTReader/odtreader.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +import argparse +import sys +import xml.etree.ElementTree as ET +from zipfile import ZipFile + +if sys.platform == "win32": + import win32_unicode_argv + + +def textOrTail(elem): + total = "" + tort = elem.text or elem.tail + if tort: + total += tort + for child in elem: + total += textOrTail(child) + return total + + +def odtToText(odtPath): + with ZipFile(odtPath, "r") as odtArchive: + try: + with odtArchive.open("content.xml") as f: + odtContent = f.read() + except Exception as e: + print("Could not find 'content.xml': {}".format(str(e))) + return + + root = ET.fromstring(odtContent) + total = "" + for child in root.find("{urn:oasis:names:tc:opendocument:xmlns:office:1.0}body").find( + "{urn:oasis:names:tc:opendocument:xmlns:office:1.0}text" + ): + if child.tag == "{urn:oasis:names:tc:opendocument:xmlns:text:1.0}p": + total += textOrTail(child) + "\n" + if total and total[-1] == "\n": + total = total[:-1] + return total + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("odtPath", help="Path to the .odt file to read") + parser.add_argument( + "-o", + "--out", + help="If the output is to be written to a file, path to the file (otherwise STDOUT is used)", + ) + args = parser.parse_args() + + output = odtToText(args.odtPath) + if args.out: + with open(args.out, "w") as outFile: + outFile.write(output) + else: + print(output) diff --git a/misp_modules/lib/ODTReader/win32_unicode_argv.py b/misp_modules/lib/ODTReader/win32_unicode_argv.py new file mode 100644 index 000000000..0619974e2 --- /dev/null +++ b/misp_modules/lib/ODTReader/win32_unicode_argv.py @@ -0,0 +1,52 @@ +""" +win32_unicode_argv.py + +Importing this will replace sys.argv with a full Unicode form. +Windows only. + +From this site, with adaptations: + http://code.activestate.com/recipes/572200/ + +Usage: simply import this module into a script. sys.argv is changed to +be a list of Unicode strings. +""" + +import sys + +# PY3 changed range to xrange +try: + range = xrange +except NameError: + pass + + +def win32_unicode_argv(): + """Uses shell32.GetCommandLineArgvW to get sys.argv as a list of Unicode + strings. + + Versions 2.x of Python don't support Unicode in sys.argv on + Windows, with the underlying Windows API instead replacing multi-byte + characters with '?'. + """ + + from ctypes import POINTER, byref, c_int, cdll, windll + from ctypes.wintypes import LPCWSTR, LPWSTR + + GetCommandLineW = cdll.kernel32.GetCommandLineW + GetCommandLineW.argtypes = [] + GetCommandLineW.restype = LPCWSTR + + CommandLineToArgvW = windll.shell32.CommandLineToArgvW + CommandLineToArgvW.argtypes = [LPCWSTR, POINTER(c_int)] + CommandLineToArgvW.restype = POINTER(LPWSTR) + + cmd = GetCommandLineW() + argc = c_int(0) + argv = CommandLineToArgvW(cmd, byref(argc)) + if argc.value > 0: + # Remove Python executable and commands if present + start = argc.value - len(sys.argv) + return [argv[i] for i in range(start, argc.value)] + + +sys.argv = win32_unicode_argv() diff --git a/misp_modules/lib/__init__.py b/misp_modules/lib/__init__.py index dffa25502..e69de29bb 100644 --- a/misp_modules/lib/__init__.py +++ b/misp_modules/lib/__init__.py @@ -1,4 +0,0 @@ -import joe_mapping -from .vt_graph_parser import * # noqa - -all = ['joe_parser', 'lastline_api', 'cof2misp', 'qintel_helper'] diff --git a/misp_modules/lib/_vmray/parser.py b/misp_modules/lib/_vmray/parser.py index 23a17e1a9..0e880de54 100644 --- a/misp_modules/lib/_vmray/parser.py +++ b/misp_modules/lib/_vmray/parser.py @@ -1,7 +1,6 @@ import base64 import json import re - from abc import ABC, abstractmethod from dataclasses import asdict, dataclass, field from enum import Enum @@ -12,7 +11,6 @@ from .rest_api import VMRayRESTAPI, VMRayRESTAPIError - USER_RE = re.compile(r".:.Users\\(.*?)\\", re.IGNORECASE) DOC_RE = re.compile(r".:.DOCUME~1.\\(.*?)\\", re.IGNORECASE) DOC_AND_SETTINGS_RE = re.compile(r".:.Documents and Settings\\(.*?)\\", re.IGNORECASE) @@ -88,9 +86,7 @@ def to_misp_object(self, tag: bool) -> MISPObject: obj = MISPObject(name="domain-ip") classifications = classifications_to_str(self.classifications) - attr = obj.add_attribute( - "domain", value=self.domain, to_ids=self.is_ioc, comment=classifications - ) + attr = obj.add_attribute("domain", value=self.domain, to_ids=self.is_ioc, comment=classifications) if tag and attr: self.tag_artifact_attribute(attr) @@ -123,9 +119,7 @@ class EmailArtifact(Artifact): def to_attributes(self) -> Iterator[Attribute]: if self.sender: classifications = classifications_to_str(self.classifications) - yield Attribute( - type="email-src", value=self.sender, comment=classifications - ) + yield Attribute(type="email-src", value=self.sender, comment=classifications) if self.subject: yield Attribute(type="email-subject", value=self.subject, to_ids=False) @@ -138,9 +132,7 @@ def to_misp_object(self, tag: bool) -> MISPObject: if self.sender: classifications = classifications_to_str(self.classifications) - attr = obj.add_attribute( - "from", value=self.sender, to_ids=self.is_ioc, comment=classifications - ) + attr = obj.add_attribute("from", value=self.sender, to_ids=self.is_ioc, comment=classifications) if tag and attr: self.tag_artifact_attribute(attr) @@ -212,13 +204,11 @@ def to_misp_object(self, tag: bool) -> MISPObject: ("sha256", self.sha256), ("ssdeep", self.ssdeep), ] - for (key, value) in hashes: + for key, value in hashes: if not value: continue - attr = obj.add_attribute( - key, value=value, to_ids=self.is_ioc, comment=classifications - ) + attr = obj.add_attribute(key, value=value, to_ids=self.is_ioc, comment=classifications) if tag and attr: self.tag_artifact_attribute(attr) @@ -274,9 +264,7 @@ def to_misp_object(self, tag: bool) -> MISPObject: obj = MISPObject(name="ip-port") classifications = classifications_to_str(self.classifications) - attr = obj.add_attribute( - "ip", value=self.ip, comment=classifications, to_ids=self.is_ioc - ) + attr = obj.add_attribute("ip", value=self.ip, comment=classifications, to_ids=self.is_ioc) if tag and attr: self.tag_artifact_attribute(attr) @@ -365,14 +353,10 @@ def to_misp_object(self, tag: bool) -> MISPObject: obj.add_attribute("pid", value=self.pid, category="External analysis") if self.parent_pid: - obj.add_attribute( - "parent-pid", value=self.parent_pid, category="External analysis" - ) + obj.add_attribute("parent-pid", value=self.parent_pid, category="External analysis") classifications = classifications_to_str(self.classifications) - name_attr = obj.add_attribute( - "name", self.filename, category="External analysis", comment=classifications - ) + name_attr = obj.add_attribute("name", self.filename, category="External analysis", comment=classifications) cmd_attr = obj.add_attribute("command-line", value=self.cmd_line) @@ -417,9 +401,7 @@ def to_misp_object(self, tag: bool) -> MISPObject: if self.operations: operations = "Operations: " + ", ".join(self.operations) - attr = obj.add_attribute( - "key", value=self.key, to_ids=self.is_ioc, comment=operations - ) + attr = obj.add_attribute("key", value=self.key, to_ids=self.is_ioc, comment=operations) if tag and attr: self.tag_artifact_attribute(attr) @@ -470,9 +452,7 @@ def to_misp_object(self, tag: bool) -> MISPObject: self.tag_artifact_attribute(attr) if self.domain: - obj.add_attribute( - "domain", self.domain, category="External analysis", to_ids=False - ) + obj.add_attribute("domain", self.domain, category="External analysis", to_ids=False) for ip in self.ips: obj.add_attribute("ip", ip, category="External analysis", to_ids=False) @@ -558,9 +538,7 @@ def vtis(self) -> Iterator[VTI]: class Summary(ReportParser): - def __init__( - self, analysis_id: int, api: VMRayRESTAPI = None, report: Dict[str, Any] = None - ): + def __init__(self, analysis_id: int, api: VMRayRESTAPI = None, report: Dict[str, Any] = None): self.analysis_id = analysis_id if report: @@ -767,9 +745,7 @@ def mitre_attacks(self) -> Iterator[MitreAttack]: techniques = mitre_attack.get("techniques", []) for technique in techniques: - mitre_attack = MitreAttack( - description=technique["description"], id=technique["id"] - ) + mitre_attack = MitreAttack(description=technique["description"], id=technique["id"]) yield mitre_attack def sandbox_type(self) -> str: @@ -799,9 +775,7 @@ def vtis(self) -> Iterator[VTI]: class SummaryV2(ReportParser): - def __init__( - self, analysis_id: int, api: VMRayRESTAPI = None, report: Dict[str, Any] = None - ): + def __init__(self, analysis_id: int, api: VMRayRESTAPI = None, report: Dict[str, Any] = None): self.analysis_id = analysis_id if report: @@ -815,9 +789,7 @@ def __init__( ) self.report = json.load(data) - def _resolve_refs( - self, data: Union[List[Dict[str, Any]], Dict[str, Any]] - ) -> Iterator[Dict[str, Any]]: + def _resolve_refs(self, data: Union[List[Dict[str, Any]], Dict[str, Any]]) -> Iterator[Dict[str, Any]]: if not data: return [] @@ -1100,10 +1072,8 @@ def from_api(self, config: Dict[str, Any]) -> None: self._setup_optional_config(config) self.report_version = self._get_report_version() - def from_base64_string( - self, config: Dict[str, Any], data: str, filename: str - ) -> None: - """ read base64 encoded summary json """ + def from_base64_string(self, config: Dict[str, Any], data: str, filename: str) -> None: + """read base64 encoded summary json""" buffer = base64.b64decode(data) self.report = json.loads(buffer) @@ -1126,9 +1096,7 @@ def _setup_optional_config(self, config: Dict[str, Any]) -> None: self.include_all_artifacts = bool(int(config.get("Artifacts", "0"))) self.include_analysis_details = bool(int(config.get("Analysis Details", "1"))) - self.use_misp_object = not self._config_from_string( - config.get("disable_misp_objects") - ) + self.use_misp_object = not self._config_from_string(config.get("disable_misp_objects")) self.tag_objects = not self._config_from_string(config.get("disable_tags")) @staticmethod @@ -1200,15 +1168,12 @@ def _online_reports(self) -> Iterator[Tuple[ReportParser, str]]: try: self._vmary_api_call(f"/rest/sample/{self.sample_id}") except VMRayRESTAPIError: - raise VMRayParseError( - f"Could not find sample id `{self.sample_id}` on server." - ) + raise VMRayParseError(f"Could not find sample id `{self.sample_id}` on server.") # check if all submission are finished if not self.ignore_analysis_finished and not self._analysis_finished(): raise VMRayParseError( - f"Not all analysis for `{self.sample_id}` are finished. " - "Try it again in a few minutes." + f"Not all analysis for `{self.sample_id}` are finished. Try it again in a few minutes." ) analysis_results = self._get_analysis() @@ -1244,9 +1209,7 @@ def _reports(self) -> Iterator[Tuple[ReportParser, Optional[str]]]: def _get_sample_verdict(self) -> Optional[str]: if self.report: if self.report_version == ReportVersion.v2: - verdict = SummaryV2.convert_verdict( - self.report["analysis_metadata"]["verdict"] - ) + verdict = SummaryV2.convert_verdict(self.report["analysis_metadata"]["verdict"]) return verdict return None @@ -1262,7 +1225,7 @@ def _get_sample_verdict(self) -> Optional[str]: return None def parse(self) -> None: - """ Convert analysis results to MISP Objects """ + """Convert analysis results to MISP Objects""" if self.use_misp_object: self.parse_as_misp_object() @@ -1296,9 +1259,7 @@ def parse_as_attributes(self) -> None: self.attributes.append(attr) for artifact in report.artifacts(): - if self.include_all_artifacts or ( - self.include_iocs and artifact.is_ioc - ): + if self.include_all_artifacts or (self.include_iocs and artifact.is_ioc): for attr in artifact.to_attributes(): self.attributes.append(attr) @@ -1327,12 +1288,8 @@ def parse_as_misp_object(self): obj.add_attribute("permalink", permalink) if self.include_report and self.report: - report_data = base64.b64encode( - json.dumps(self.report, indent=2).encode("utf-8") - ).decode("utf-8") - obj.add_attribute( - "sandbox-file", value=self.report_name, data=report_data - ) + report_data = base64.b64encode(json.dumps(self.report, indent=2).encode("utf-8")).decode("utf-8") + obj.add_attribute("sandbox-file", value=self.report_name, data=report_data) score = report.score() attr_score = obj.add_attribute("score", score) @@ -1355,9 +1312,7 @@ def parse_as_misp_object(self): vtis.append(vti) for artifact in report.artifacts(): - if self.include_all_artifacts or ( - self.include_iocs and artifact.is_ioc - ): + if self.include_all_artifacts or (self.include_iocs and artifact.is_ioc): if artifact not in artifacts: artifacts.append(artifact) else: @@ -1397,7 +1352,7 @@ def parse_as_misp_object(self): self.event.add_tag(f'vmray:verdict="{verdict}"') def to_json(self) -> Dict[str, Any]: - """ Convert parsed results into JSON """ + """Convert parsed results into JSON""" if not self.use_misp_object: results = [] diff --git a/misp_modules/lib/_vmray/rest_api.py b/misp_modules/lib/_vmray/rest_api.py index d37c6f214..2d326c8fd 100644 --- a/misp_modules/lib/_vmray/rest_api.py +++ b/misp_modules/lib/_vmray/rest_api.py @@ -4,24 +4,9 @@ import base64 import datetime import os.path -import requests import urllib.parse -# disable nasty certification warning -# pylint: disable=no-member -try: - requests.packages.urllib3.disable_warnings() -except AttributeError: - try: - import urllib3 - try: - urllib3.disable_warnings() - except AttributeError: - pass - except ImportError: - pass - -# pylint: disable= +import requests class VMRayRESTAPIError(Exception): @@ -39,9 +24,15 @@ def handle_rest_api_result(result): try: json_result = result.json() except ValueError: - raise VMRayRESTAPIError("API returned error %u: %s" % (result.status_code, result.text), status_code=result.status_code) + raise VMRayRESTAPIError( + "API returned error %u: %s" % (result.status_code, result.text), + status_code=result.status_code, + ) - raise VMRayRESTAPIError(json_result.get("error_msg", "Unknown error"), status_code=result.status_code) + raise VMRayRESTAPIError( + json_result.get("error_msg", "Unknown error"), + status_code=result.status_code, + ) class VMRayRESTAPI(object): @@ -72,10 +63,7 @@ def call(self, http_method, api_path, params=None, raw_data=False): if params is not None: for key, value in params.items(): - if isinstance(value, (datetime.date, - datetime.datetime, - float, - int)): + if isinstance(value, (datetime.date, datetime.datetime, float, int)): req_params[key] = str(value) elif isinstance(value, str): req_params[key] = str(value) @@ -97,7 +85,7 @@ def call(self, http_method, api_path, params=None, raw_data=False): req_params[b64_key] = b64_value file_params[key] = (filename, value, "application/octet-stream") else: - raise VMRayRESTAPIError("Parameter \"%s\" has unknown type \"%s\"" % (key, type(value))) + raise VMRayRESTAPIError('Parameter "%s" has unknown type "%s"' % (key, type(value))) # construct request if file_params: @@ -113,7 +101,15 @@ def call(self, http_method, api_path, params=None, raw_data=False): req_data = None # do request - result = requests_func(self.server + api_path, data=req_data, params=req_params, headers={"Authorization": "api_key " + self.api_key}, files=files, verify=self.verify_cert, stream=raw_data) + result = requests_func( + self.server + api_path, + data=req_data, + params=req_params, + headers={"Authorization": "api_key " + self.api_key}, + files=files, + verify=self.verify_cert, + stream=raw_data, + ) handle_rest_api_result(result) if raw_data: @@ -134,7 +130,11 @@ def call(self, http_method, api_path, params=None, raw_data=False): # get cached results while "continuation_id" in json_result: # send request to server - result = requests.get("%s/rest/continuation/%u" % (self.server, json_result["continuation_id"]), headers={"Authorization": "api_key " + self.api_key}, verify=self.verify_cert) + result = requests.get( + "%s/rest/continuation/%u" % (self.server, json_result["continuation_id"]), + headers={"Authorization": "api_key " + self.api_key}, + verify=self.verify_cert, + ) handle_rest_api_result(result) # parse result diff --git a/misp_modules/lib/cof2misp/cof.py b/misp_modules/lib/cof2misp/cof.py index d7420a0d3..4acf104a4 100644 --- a/misp_modules/lib/cof2misp/cof.py +++ b/misp_modules/lib/cof2misp/cof.py @@ -12,6 +12,7 @@ import ipaddress import sys + import ndjson @@ -27,7 +28,10 @@ def is_valid_ip(ip: str) -> bool: try: ipaddress.ip_address(ip) except Exception as ex: - print("is_valid_ip(%s) returned False. Reason: %s" % (ip, str(ex)), file = sys.stderr) + print( + "is_valid_ip(%s) returned False. Reason: %s" % (ip, str(ex)), + file=sys.stderr, + ) return False return True @@ -51,29 +55,31 @@ def is_cof_valid_simple(d: dict) -> bool: """ if "rrname" not in d: - print("Missing MANDATORY field 'rrname'", file = sys.stderr) + print("Missing MANDATORY field 'rrname'", file=sys.stderr) return False - if not isinstance(d['rrname'], str): - print("Type error: 'rrname' is not a JSON string", file = sys.stderr) + if not isinstance(d["rrname"], str): + print("Type error: 'rrname' is not a JSON string", file=sys.stderr) return False if "rrtype" not in d: - print("Missing MANDATORY field 'rrtype'", file = sys.stderr) + print("Missing MANDATORY field 'rrtype'", file=sys.stderr) return False - if not isinstance(d['rrtype'], str): - print("Type error: 'rrtype' is not a JSON string", file = sys.stderr) + if not isinstance(d["rrtype"], str): + print("Type error: 'rrtype' is not a JSON string", file=sys.stderr) return False if "rdata" not in d: - print("Missing MANDATORY field 'rdata'", file = sys.stderr) + print("Missing MANDATORY field 'rdata'", file=sys.stderr) return False if "rdata" not in d: - print("Missing MANDATORY field 'rdata'", file = sys.stderr) + print("Missing MANDATORY field 'rdata'", file=sys.stderr) return False - if not isinstance(d['rdata'], str) and not isinstance(d['rdata'], list): - print("'rdata' is not a list and not a string.", file = sys.stderr) + if not isinstance(d["rdata"], str) and not isinstance(d["rdata"], list): + print("'rdata' is not a list and not a string.", file=sys.stderr) return False if not ("time_first" in d and "time_last" in d) or ("zone_time_first" in d and "zone_time_last" in d): - print("We are missing EITHER ('first_seen' and 'last_seen') OR ('zone_time_first' and zone_time_last') fields", - file = sys.stderr) + print( + "We are missing EITHER ('first_seen' and 'last_seen') OR ('zone_time_first' and zone_time_last') fields", + file=sys.stderr, + ) return False # currently we don't check the OPTIONAL fields. Sorry... to be done later. return True @@ -101,16 +107,16 @@ def validate_dnsdbflex(d: dict, strict=True) -> bool: { "rrtype": , "rrname": } """ if "rrname" not in d: - print("Missing MANDATORY field 'rrname'", file = sys.stderr) + print("Missing MANDATORY field 'rrname'", file=sys.stderr) return False - if not isinstance(d['rrname'], str): - print("Type error: 'rrname' is not a JSON string", file = sys.stderr) + if not isinstance(d["rrname"], str): + print("Type error: 'rrname' is not a JSON string", file=sys.stderr) return False if "rrtype" not in d: - print("Missing MANDATORY field 'rrtype'", file = sys.stderr) + print("Missing MANDATORY field 'rrtype'", file=sys.stderr) return False - if not isinstance(d['rrtype'], str): - print("Type error: 'rrtype' is not a JSON string", file = sys.stderr) + if not isinstance(d["rrtype"], str): + print("Type error: 'rrtype' is not a JSON string", file=sys.stderr) return False return True @@ -118,33 +124,37 @@ def validate_dnsdbflex(d: dict, strict=True) -> bool: if __name__ == "__main__": # simple, poor man's unit tests. - print(80 * "=", file = sys.stderr) - print("Unit Tests:", file = sys.stderr) + print(80 * "=", file=sys.stderr) + print("Unit Tests:", file=sys.stderr) assert not is_valid_ip("a.2.3.4") assert is_valid_ip("99.88.77.6") assert is_valid_ip("2a0c:88:77:6::1") # COF validation - print(80 * "=", file = sys.stderr) - print("COF unit tests....", file = sys.stderr) + print(80 * "=", file=sys.stderr) + print("COF unit tests....", file=sys.stderr) mock_input = """{"count":1909,"rdata":["cpa.circl.lu"],"rrname":"www.circl.lu","rrtype":"CNAME","time_first":"1315586409","time_last":"1449566799"} {"count":2560,"rdata":["cpab.circl.lu"],"rrname":"www.circl.lu","rrtype":"CNAME","time_first":"1449584660","time_last":"1617676151"}""" i = 0 for entry in ndjson.loads(mock_input): - retval = validate_cof(entry, strict = False) + retval = validate_cof(entry, strict=False) assert retval print("line %d is valid: %s" % (i, retval)) i += 1 - test2 = '{"count": 2, "time_first": 1619556027, "time_last": 1619556034, "rrname": "westernunion.com.ph.unblock-all.com.beta.opera-mini.net.", "rrtype": "A", "bailiwick": "beta.opera-mini.net.", "rdata": ["185.26.181.253"]}' + test2 = ( + '{"count": 2, "time_first": 1619556027, "time_last": 1619556034, "rrname":' + ' "westernunion.com.ph.unblock-all.com.beta.opera-mini.net.", "rrtype": "A", "bailiwick":' + ' "beta.opera-mini.net.", "rdata": ["185.26.181.253"]}' + ) for entry in ndjson.loads(test2): assert validate_cof(entry) # dnsdbflex validation - print(80 * "=", file = sys.stderr) - print("dnsdbflex unit tests....", file = sys.stderr) + print(80 * "=", file=sys.stderr) + print("dnsdbflex unit tests....", file=sys.stderr) mock_input = """{"rrname":"labs.deep-insights.ai.","rrtype":"A"} {"rrname":"www.deep-insights.ca.","rrtype":"CNAME"} @@ -155,11 +165,10 @@ def validate_dnsdbflex(d: dict, strict=True) -> bool: i = 0 for entry in ndjson.loads(mock_input): - retval = validate_dnsdbflex(entry, strict = False) + retval = validate_dnsdbflex(entry, strict=False) assert retval print("dnsdbflex line %d is valid: %s" % (i, retval)) i += 1 - - print(80 * "=", file = sys.stderr) - print("Unit Tests DONE", file = sys.stderr) + print(80 * "=", file=sys.stderr) + print("Unit Tests DONE", file=sys.stderr) diff --git a/misp_modules/lib/dnstrails/LICENSE b/misp_modules/lib/dnstrails/LICENSE new file mode 100644 index 000000000..0da36c5e7 --- /dev/null +++ b/misp_modules/lib/dnstrails/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 Sebastien Larinier + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/misp_modules/lib/dnstrails/__init__.py b/misp_modules/lib/dnstrails/__init__.py new file mode 100644 index 000000000..9d56f81d1 --- /dev/null +++ b/misp_modules/lib/dnstrails/__init__.py @@ -0,0 +1,2 @@ +from dnstrails.api import DnsTrails +from dnstrails.exception import APIError diff --git a/misp_modules/lib/dnstrails/api.py b/misp_modules/lib/dnstrails/api.py new file mode 100644 index 000000000..880552c6d --- /dev/null +++ b/misp_modules/lib/dnstrails/api.py @@ -0,0 +1,424 @@ +import warnings + +import requests +from dnstrails.exception import APIError + +""" +dnstrails.api +~~~~~~~~~~~~~ + +This module implements the DNSTrail API. + +:copyright: (c) 2018 - by Sebastien Larinier +""" + + +def deprecated(message): + def deprecated_decorator(func): + def deprecated_func(*args, **kwargs): + warnings.warn( + "{} is a deprecated function. {}".format(func.__name__, message), + category=DeprecationWarning, + stacklevel=2, + ) + warnings.simplefilter("default", DeprecationWarning) + return func(*args, **kwargs) + + return deprecated_func + + return deprecated_decorator + + +class DnsTrails: + """Wrapper around the DNSTrail REST + + :param key: The DNSTrail API key that can be obtained from your account + page (https://securitytrails.com/) + :type key: str + + :param version Version of API + """ + + def __init__(self, api_key, version="v1"): + self._key = api_key + self.base_url = "https://api.securitytrails.com" + self.version = version + self._session = requests.Session() + + self.methods = {"get": self._session.get, "post": self._session.post} + + def _prepare_query(self, query, **kwargs): + + self._headers = {"apikey": self._key} + self._payload = {} + if "page" in kwargs: + self._payload["page"] = kwargs.get("page") + if "mask" in kwargs: + self._payload["mask"] = kwargs.get("mask") + + self.data = {"filter": {k: v for k, v in kwargs.items() if k != "mask" and k != "page"}} + + self.url = "%s/%s/%s" % (self.base_url, self.version, "/".join(query)) + + def _request(self, method="get"): + + data = None + + try: + if method == "get": + if not self._payload: + response = self.methods[method](self.url, headers=self._headers) + else: + + response = self.methods[method](self.url, headers=self._headers, params=self._payload) + elif method == "post": + self._headers["Content-type"] = "application/json" + + response = self.methods[method]( + self.url, + headers=self._headers, + json=self.data, + params=self._payload, + ) + + except Exception as e: + raise APIError("Unable to connect DNSTrail %s" % e) + + if response.status_code == requests.codes.NOT_FOUND: + + raise APIError("Page Not found %s" % self.url) + elif response.status_code == requests.codes.FORBIDDEN: + raise APIError("Access Forbidden") + elif response.status_code != requests.codes.OK: + try: + error = response.json()["message"] + except Exception as e: + error = "Invalid API key %s" % e + + raise APIError(error) + + try: + + data = response.json() + + except Exception as e: + raise APIError("Unable to parse JSON %s" % e) + + return data + + def _query(self, query, method="get", **kwargs): + self._prepare_query(query, **kwargs) + + data = self._request(method=method) + + if data: + return data + else: + raise APIError("Error API") + + def ping(self): + """Call API Ping to test your API key and connectivity + GET https://api.securitytrails.com/v1/ping + :returns: dict -- status of connectivity + + """ + query = ["ping"] + return self._query(query) + + def domain(self, domain): + """Call API Get Domain information about a domain + GET https://api.securitytrails.com/v1/domain/ + :param domain: fqdn for query + :type: domain: str + :return: dict -- a dictionary containing the result of the service on + one domain + """ + query = ["domain", domain] + return self._query(query) + + def subdomains(self, domain): + """Call API subdomains on one domain + GET https://api.securitytrails.com/v1/domain//subdomains + :param domain: domain for query + :return: dict -- a dictionary containing the list of subdomains + """ + query = ["domain", domain, "subdomains"] + + return self._query(query) + + def tags(self, domain): + """Call API tags for listing tags about a domain + GET https://api.securitytrails.com/v1/domain//tags + :param domain: domain for query + :type: domain: str + :return: dict -- a dictionary containing the list of tags + """ + query = ["domain", domain, "tags"] + + return self._query(query) + + def whois(self, domain): + """Call API Whois on a domain + GET https://api.securitytrails.com/v1/domain//whois + :param domain: domain for query + :type domain: str + :return: dict -- a dictionary containing the whois result of a domain + """ + + query = ["domain", domain, "whois"] + + return self._query(query) + + def history_dns_ipv4(self, domain, **kwargs): + """Call API historical Ipv4 on a domain + GET https://api.securitytrails.com/v1/history//dns/a?page= + :param domain: domain for query + :type domain: str + :param page: number of page of the result + :type page: int + :return: dict -- a dictionary containing the Ipv4 historical data + """ + query = ["history", domain, "dns", "a"] + + return self._query(query, **kwargs) + + def history_dns_aaaa(self, domain, **kwargs): + """Call API historical Ipv6 on a domain + GET https://api.securitytrails.com/v1/history//dns/aaaa?page= + :param domain: domain for query + :type:domain + :param page: number of page of the result + :type page: int + :return: dict -- a dictionary containing the Ipv6 historical data + """ + query = ["history", domain, "dns", "aaaa"] + + return self._query(query, **kwargs) + + def history_dns_mx(self, domain, **kwargs): + """Call API historical mx on a domain + GET https://api.securitytrails.com/v1/history//dns/mx?page= + :param domain: domain for query + :param page: number of page of the result + :type page: int + :return: dict -- a dictionary containing the mx historical data + """ + query = ["history", domain, "dns", "mx"] + + return self._query(query, **kwargs) + + def history_dns_ns(self, domain, **kwargs): + """Call API historical mx on a domain + GET https://api.securitytrails.com/v1/history//dns/ns?page= + :param domain: domain for query + :type domain: str + :param page: number of page of the result + :type page: int + :return: dict -- a dictionary containing the ns historical data + """ + query = ["history", domain, "dns", "ns"] + + return self._query(query, **kwargs) + + def history_dns_soa(self, domain, **kwargs): + """Call API historical mx on a domain + GET https://api.securitytrails.com/v1/history//dns/soa?page= + :param domain: domain for query + :type domain: str + :param page: number of page of the result + :type page: int + :return: dict -- a dictionary containing the soa historical data + """ + query = ["history", domain, "dns", "soa"] + + return self._query(query, **kwargs) + + def history_dns_txt(self, domain, **kwargs): + """Call API historical txt on a domain + GET https://api.securitytrails.com/v1/history//dns/txt?page= + :param domain: domain for query + :type domain: str + :param page: number of page of the result + :type page: int + :return: dict -- a dictionary containing the txt historical data + """ + query = ["history", domain, "dns", "txt"] + + return self._query(query, **kwargs) + + def history_whois(self, domain, **kwargs): + """Call API historical whois on a domain + GET https://api.securitytrails.com/v1/history//whois?page= + :param domain: domain for query + :type domain: str + :param page: number of page of the result + :type page: int + :return: dict -- a dictionary containing the whois historical data + """ + + query = ["history", domain, "whois"] + + return self._query(query, **kwargs) + + def explore_ip(self, ip, **kwargs): + """Call API explore IP to have he neighbors in any given IP level range + and essentially allows you to explore closeby IP addresses. + GET https://api.securitytrails.com/v1/explore/ip/ip?mask= + :param ip: Ipv4 for query + :type ip: str + :param mask: mask of the block + :type mask: int + :return: dict -- a dictionary containing the neighboors of IP + """ + query = ["explore", "ip", ip] + + return self._query(query, **kwargs) + + def searching_domains(self, **kwargs): + """Call API searching domain + POST https://api.securitytrails.com/v1/search/list?page= + :param page: page results + :type page: int + + :param ipv4 (can include a network mask): + :type ipv4: str + + :param ipv6: + :type ipv6: str + + :param mx: + :type ipv6: str + + :param ns: + :type ns: str + + :param cname: + :type cname: str + + :param subdomain: + :type subdomain: str + + :param apex_domain: + :type subdomain: str + + :param soa_email: + :type soa_email: str + + :param tld: + :type tld: str + + :param whois_email: + :type whois_email: str + + :param whois_street1: + :type whois_street1: str + + :param whois_street2: + :type whois_street2: str + + :param whois_street3: + :type whois_street3: str + + :param whois_street4: + :type whois_street4: str + + :param whois_telephone: + :type whois_telephone: str + + :param whois_postalCode: + :type whois_postalCode: str + + :param whois_organization: + :type whois_organization: str + + :param whois_name: + :type whois_name: str + + :param whois_fax: + :type whois_fax: str + + :param whois_city: + :type whois_city: str + + :param keyword: + :type keyword: str + + :return: dict -- a dictionary containing the results of domain searching + """ + query = ["search", "list"] + + return self._query(query, method="post", **kwargs) + + @deprecated("a new implementation of stat will be developped") + def search_stats(self, **kwargs): + """Call API stats + POST https://api.securitytrails.com/v1/search/list/stats + + :param ipv4 (can include a network mask): + :type ipv4: str + + :param ipv6: + :type ipv6: str + + :param mx: + :type ipv6: str + + :param ns: + :type ns: str + + :param cname: + :type cname: str + + :param subdomain: + :type subdomain: str + + :param apex_domain: + :type subdomain: str + + :param soa_email: + :type soa_email: str + + :param tld: + :type tld: str + + :param whois_email: + :type whois_email: str + + :param whois_street1: + :type whois_street1: str + + :param whois_street2: + :type whois_street2: str + + :param whois_street3: + :type whois_street3: str + + :param whois_street4: + :type whois_street4: str + + :param whois_telephone: + :type whois_telephone: str + + :param whois_postalCode: + :type whois_postalCode: str + + :param whois_organization: + :type whois_organization: str + + :param whois_name: + :type whois_name: str + + :param whois_fax: + :type whois_fax: str + + :param whois_city: + :type whois_city: str + + :param keyword: + :type keyword: str + + :return: dict -- a dictionary containing the results of domain searching + """ + query = ["search", "list", "stats"] + + return self._query(query, method="post", **kwargs) diff --git a/misp_modules/lib/dnstrails/exception.py b/misp_modules/lib/dnstrails/exception.py new file mode 100644 index 000000000..1f5c0d5c8 --- /dev/null +++ b/misp_modules/lib/dnstrails/exception.py @@ -0,0 +1,7 @@ +class APIError(Exception): + + def __init__(self, value): + self.value = value + + def __str__(self): + return self.value diff --git a/misp_modules/lib/joe_mapping.py b/misp_modules/lib/joe_mapping.py index eda961ebe..f52341454 100644 --- a/misp_modules/lib/joe_mapping.py +++ b/misp_modules/lib/joe_mapping.py @@ -1,114 +1,100 @@ arch_type_mapping = { - 'ANDROID': 'parse_apk', - 'LINUX': 'parse_elf', - 'WINDOWS': 'parse_pe' + "ANDROID": "parse_apk", + "LINUX": "parse_elf", + "WINDOWS": "parse_pe", } domain_object_mapping = { - '@ip': {'type': 'ip-dst', 'object_relation': 'ip'}, - '@name': {'type': 'domain', 'object_relation': 'domain'} + "@ip": {"type": "ip-dst", "object_relation": "ip"}, + "@name": {"type": "domain", "object_relation": "domain"}, } dropped_file_mapping = { - '@entropy': {'type': 'float', 'object_relation': 'entropy'}, - '@file': {'type': 'filename', 'object_relation': 'filename'}, - '@size': {'type': 'size-in-bytes', 'object_relation': 'size-in-bytes'}, - '@type': {'type': 'mime-type', 'object_relation': 'mimetype'} + "@entropy": {"type": "float", "object_relation": "entropy"}, + "@file": {"type": "filename", "object_relation": "filename"}, + "@size": {"type": "size-in-bytes", "object_relation": "size-in-bytes"}, + "@type": {"type": "mime-type", "object_relation": "mimetype"}, } dropped_hash_mapping = { - 'MD5': 'md5', - 'SHA': 'sha1', - 'SHA-256': 'sha256', - 'SHA-512': 'sha512' + "MD5": "md5", + "SHA": "sha1", + "SHA-256": "sha256", + "SHA-512": "sha512", } elf_object_mapping = { - 'epaddr': 'entrypoint-address', - 'machine': 'arch', - 'osabi': 'os_abi' + "epaddr": "entrypoint-address", + "machine": "arch", + "osabi": "os_abi", } elf_section_flags_mapping = { - 'A': 'ALLOC', - 'I': 'INFO_LINK', - 'M': 'MERGE', - 'S': 'STRINGS', - 'T': 'TLS', - 'W': 'WRITE', - 'X': 'EXECINSTR' -} -file_object_fields = ( - 'filename', - 'md5', - 'sha1', - 'sha256', - 'sha512', - 'ssdeep' -) + "A": "ALLOC", + "I": "INFO_LINK", + "M": "MERGE", + "S": "STRINGS", + "T": "TLS", + "W": "WRITE", + "X": "EXECINSTR", +} +file_object_fields = ("filename", "md5", "sha1", "sha256", "sha512", "ssdeep") file_object_mapping = { - 'entropy': {'type': 'float', 'object_relation': 'entropy'}, - 'filesize': {'type': 'size-in-bytes', 'object_relation': 'size-in-bytes'}, - 'filetype': {'type': 'mime-type', 'object_relation': 'mimetype'} + "entropy": {"type": "float", "object_relation": "entropy"}, + "filesize": {"type": "size-in-bytes", "object_relation": "size-in-bytes"}, + "filetype": {"type": "mime-type", "object_relation": "mimetype"}, } file_references_mapping = { - 'fileCreated': 'creates', - 'fileDeleted': 'deletes', - 'fileMoved': 'moves', - 'fileRead': 'reads', - 'fileWritten': 'writes' + "fileCreated": "creates", + "fileDeleted": "deletes", + "fileMoved": "moves", + "fileRead": "reads", + "fileWritten": "writes", } -network_behavior_fields = ('srcip', 'dstip', 'srcport', 'dstport') +network_behavior_fields = ("srcip", "dstip", "srcport", "dstport") network_connection_object_mapping = { - 'srcip': {'type': 'ip-src', 'object_relation': 'ip-src'}, - 'dstip': {'type': 'ip-dst', 'object_relation': 'ip-dst'}, - 'srcport': {'type': 'port', 'object_relation': 'src-port'}, - 'dstport': {'type': 'port', 'object_relation': 'dst-port'} + "srcip": {"type": "ip-src", "object_relation": "ip-src"}, + "dstip": {"type": "ip-dst", "object_relation": "ip-dst"}, + "srcport": {"type": "port", "object_relation": "src-port"}, + "dstport": {"type": "port", "object_relation": "dst-port"}, } pe_object_fields = { - 'entrypoint': {'type': 'text', 'object_relation': 'entrypoint-address'}, - 'imphash': {'type': 'imphash', 'object_relation': 'imphash'} + "entrypoint": {"type": "text", "object_relation": "entrypoint-address"}, + "imphash": {"type": "imphash", "object_relation": "imphash"}, } pe_object_mapping = { - 'CompanyName': 'company-name', - 'FileDescription': 'file-description', - 'FileVersion': 'file-version', - 'InternalName': 'internal-filename', - 'LegalCopyright': 'legal-copyright', - 'OriginalFilename': 'original-filename', - 'ProductName': 'product-filename', - 'ProductVersion': 'product-version', - 'Translation': 'lang-id' + "CompanyName": "company-name", + "FileDescription": "file-description", + "FileVersion": "file-version", + "InternalName": "internal-filename", + "LegalCopyright": "legal-copyright", + "OriginalFilename": "original-filename", + "ProductName": "product-filename", + "ProductVersion": "product-version", + "Translation": "lang-id", } pe_section_object_mapping = { - 'characteristics': {'type': 'text', 'object_relation': 'characteristic'}, - 'entropy': {'type': 'float', 'object_relation': 'entropy'}, - 'name': {'type': 'text', 'object_relation': 'name'}, - 'rawaddr': {'type': 'hex', 'object_relation': 'offset'}, - 'rawsize': {'type': 'size-in-bytes', 'object_relation': 'size-in-bytes'}, - 'virtaddr': {'type': 'hex', 'object_relation': 'virtual_address'}, - 'virtsize': {'type': 'size-in-bytes', 'object_relation': 'virtual_size'} + "characteristics": {"type": "text", "object_relation": "characteristic"}, + "entropy": {"type": "float", "object_relation": "entropy"}, + "name": {"type": "text", "object_relation": "name"}, + "rawaddr": {"type": "hex", "object_relation": "offset"}, + "rawsize": {"type": "size-in-bytes", "object_relation": "size-in-bytes"}, + "virtaddr": {"type": "hex", "object_relation": "virtual_address"}, + "virtsize": {"type": "size-in-bytes", "object_relation": "virtual_size"}, } process_object_fields = { - 'cmdline': 'command-line', - 'name': 'name', - 'parentpid': 'parent-pid', - 'pid': 'pid', - 'path': 'current-directory' -} -protocols = { - 'tcp': 4, - 'udp': 4, - 'icmp': 3, - 'http': 7, - 'https': 7, - 'ftp': 7 + "cmdline": "command-line", + "name": "name", + "parentpid": "parent-pid", + "pid": "pid", + "path": "current-directory", } +protocols = {"tcp": 4, "udp": 4, "icmp": 3, "http": 7, "https": 7, "ftp": 7} registry_references_mapping = { - 'keyValueCreated': 'creates', - 'keyValueModified': 'modifies' + "keyValueCreated": "creates", + "keyValueModified": "modifies", } regkey_object_mapping = { - 'name': {'type': 'text', 'object_relation': 'name'}, - 'newdata': {'type': 'text', 'object_relation': 'data'}, - 'path': {'type': 'regkey', 'object_relation': 'key'} + "name": {"type": "text", "object_relation": "name"}, + "newdata": {"type": "text", "object_relation": "data"}, + "path": {"type": "regkey", "object_relation": "key"}, } signerinfo_object_mapping = { - 'sigissuer': {'type': 'text', 'object_relation': 'issuer'}, - 'version': {'type': 'text', 'object_relation': 'version'} + "sigissuer": {"type": "text", "object_relation": "issuer"}, + "version": {"type": "text", "object_relation": "version"}, } diff --git a/misp_modules/lib/joe_parser.py b/misp_modules/lib/joe_parser.py index e701ff3a4..f96e5feef 100644 --- a/misp_modules/lib/joe_parser.py +++ b/misp_modules/lib/joe_parser.py @@ -2,17 +2,32 @@ import json from collections import defaultdict from datetime import datetime + +from joe_mapping import ( + arch_type_mapping, + domain_object_mapping, + dropped_file_mapping, + dropped_hash_mapping, + elf_object_mapping, + elf_section_flags_mapping, + file_object_fields, + file_object_mapping, + file_references_mapping, + network_behavior_fields, + network_connection_object_mapping, + pe_object_fields, + pe_object_mapping, + pe_section_object_mapping, + process_object_fields, + protocols, + registry_references_mapping, + regkey_object_mapping, + signerinfo_object_mapping, +) from pymisp import MISPAttribute, MISPEvent, MISPObject -from joe_mapping import (arch_type_mapping, domain_object_mapping, - dropped_file_mapping, dropped_hash_mapping, elf_object_mapping, - elf_section_flags_mapping, file_object_fields, file_object_mapping, - file_references_mapping, network_behavior_fields, - network_connection_object_mapping, pe_object_fields, pe_object_mapping, - pe_section_object_mapping, process_object_fields, protocols, - registry_references_mapping, regkey_object_mapping, signerinfo_object_mapping) -class JoeParser(): +class JoeParser: def __init__(self, config): self.misp_event = MISPEvent() self.references = defaultdict(list) @@ -54,163 +69,183 @@ def handle_attributes(self): attribute_uuid = self.create_attribute(attribute_type, attribute_value) for reference in references: source_uuid, relationship = reference - self.references[source_uuid].append(dict(referenced_uuid=attribute_uuid, - relationship_type=relationship)) + self.references[source_uuid].append( + dict( + referenced_uuid=attribute_uuid, + relationship_type=relationship, + ) + ) def parse_dropped_files(self): - droppedinfo = self.data['droppedinfo'] + droppedinfo = self.data["droppedinfo"] if droppedinfo: - for droppedfile in droppedinfo['hash']: - file_object = MISPObject('file') + for droppedfile in droppedinfo["hash"]: + file_object = MISPObject("file") for key, mapping in dropped_file_mapping.items(): if droppedfile.get(key) is not None: - attribute = {'value': droppedfile[key], 'to_ids': False} + attribute = {"value": droppedfile[key], "to_ids": False} attribute.update(mapping) file_object.add_attribute(**attribute) - if droppedfile['@malicious'] == 'true': + if droppedfile["@malicious"] == "true": file_object.add_attribute( **{ - 'type': 'text', - 'object_relation': 'state', - 'value': 'Malicious', - 'to_ids': False + "type": "text", + "object_relation": "state", + "value": "Malicious", + "to_ids": False, } ) - for h in droppedfile['value']: - hash_type = dropped_hash_mapping[h['@algo']] + for h in droppedfile["value"]: + hash_type = dropped_hash_mapping[h["@algo"]] file_object.add_attribute( **{ - 'type': hash_type, - 'object_relation': hash_type, - 'value': h['$'], - 'to_ids': False + "type": hash_type, + "object_relation": hash_type, + "value": h["$"], + "to_ids": False, } ) self.misp_event.add_object(file_object) - reference_key = (int(droppedfile['@targetid']), droppedfile['@process']) + reference_key = (int(droppedfile["@targetid"]), droppedfile["@process"]) if reference_key in self.process_references: self.references[self.process_references[reference_key]].append( { - 'referenced_uuid': file_object.uuid, - 'relationship_type': 'drops' + "referenced_uuid": file_object.uuid, + "relationship_type": "drops", } ) def parse_mitre_attack(self): - mitreattack = self.data.get('mitreattack', {}) + mitreattack = self.data.get("mitreattack", {}) if mitreattack: - for tactic in mitreattack['tactic']: - if tactic.get('technique'): - for technique in tactic['technique']: - self.misp_event.add_tag(f'misp-galaxy:mitre-attack-pattern="{technique["name"]} - {technique["id"]}"') + for tactic in mitreattack["tactic"]: + if tactic.get("technique"): + for technique in tactic["technique"]: + self.misp_event.add_tag( + f'misp-galaxy:mitre-attack-pattern="{technique["name"]} - {technique["id"]}"' + ) def parse_network_behavior(self): - network = self.data['behavior']['network'] + network = self.data["behavior"]["network"] connections = defaultdict(lambda: defaultdict(set)) for protocol, layer in protocols.items(): if network.get(protocol): - for packet in network[protocol]['packet']: - timestamp = datetime.strptime(self.parse_timestamp(packet['timestamp']), '%b %d, %Y %H:%M:%S.%f') + for packet in network[protocol]["packet"]: + timestamp = datetime.strptime( + self.parse_timestamp(packet["timestamp"]), + "%b %d, %Y %H:%M:%S.%f", + ) connections[tuple(packet.get(field) for field in network_behavior_fields)][protocol].add(timestamp) for connection, data in connections.items(): attributes = self.prefetch_attributes_data(connection) if len(data.keys()) == len(set(protocols[protocol] for protocol in data.keys())): - network_connection_object = MISPObject('network-connection') + network_connection_object = MISPObject("network-connection") for attribute in attributes: network_connection_object.add_attribute(**attribute) network_connection_object.add_attribute( **{ - 'type': 'datetime', - 'object_relation': 'first-packet-seen', - 'value': min(tuple(min(timestamp) for timestamp in data.values())), - 'to_ids': False + "type": "datetime", + "object_relation": "first-packet-seen", + "value": min(tuple(min(timestamp) for timestamp in data.values())), + "to_ids": False, } ) for protocol in data.keys(): network_connection_object.add_attribute( **{ - 'type': 'text', - 'object_relation': f'layer{protocols[protocol]}-protocol', - 'value': protocol, - 'to_ids': False + "type": "text", + "object_relation": f"layer{protocols[protocol]}-protocol", + "value": protocol, + "to_ids": False, } ) self.misp_event.add_object(network_connection_object) - self.references[self.analysisinfo_uuid].append(dict(referenced_uuid=network_connection_object.uuid, - relationship_type='initiates')) + self.references[self.analysisinfo_uuid].append( + dict( + referenced_uuid=network_connection_object.uuid, + relationship_type="initiates", + ) + ) else: for protocol, timestamps in data.items(): - network_connection_object = MISPObject('network-connection') + network_connection_object = MISPObject("network-connection") for attribute in attributes: network_connection_object.add_attribute(**attribute) network_connection_object.add_attribute( **{ - 'type': 'datetime', - 'object_relation': 'first-packet-seen', - 'value': min(timestamps), - 'to_ids': False + "type": "datetime", + "object_relation": "first-packet-seen", + "value": min(timestamps), + "to_ids": False, } ) network_connection_object.add_attribute( **{ - 'type': 'text', - 'object_relation': f'layer{protocols[protocol]}-protocol', - 'value': protocol, - 'to_ids': False + "type": "text", + "object_relation": f"layer{protocols[protocol]}-protocol", + "value": protocol, + "to_ids": False, } ) self.misp_event.add_object(network_connection_object) - self.references[self.analysisinfo_uuid].append(dict(referenced_uuid=network_connection_object.uuid, - relationship_type='initiates')) + self.references[self.analysisinfo_uuid].append( + dict( + referenced_uuid=network_connection_object.uuid, + relationship_type="initiates", + ) + ) def parse_screenshot(self): - if self.data['behavior'].get('screenshotdata', {}).get('interesting') is not None: - screenshotdata = self.data['behavior']['screenshotdata']['interesting']['$'] + if self.data["behavior"].get("screenshotdata", {}).get("interesting") is not None: + screenshotdata = self.data["behavior"]["screenshotdata"]["interesting"]["$"] self.misp_event.add_attribute( **{ - 'type': 'attachment', - 'value': 'screenshot.jpg', - 'data': screenshotdata, - 'disable_correlation': True, - 'to_ids': False + "type": "attachment", + "value": "screenshot.jpg", + "data": screenshotdata, + "disable_correlation": True, + "to_ids": False, } ) def parse_system_behavior(self): - if not 'system' in self.data['behavior']: + if not "system" in self.data["behavior"]: return - system = self.data['behavior']['system'] - if system.get('processes'): - process_activities = {'fileactivities': self.parse_fileactivities, - 'registryactivities': self.parse_registryactivities} - for process in system['processes']['process']: - general = process['general'] - process_object = MISPObject('process') + system = self.data["behavior"]["system"] + if system.get("processes"): + process_activities = { + "fileactivities": self.parse_fileactivities, + "registryactivities": self.parse_registryactivities, + } + for process in system["processes"]["process"]: + general = process["general"] + process_object = MISPObject("process") for feature, relation in process_object_fields.items(): process_object.add_attribute( **{ - 'type': 'text', - 'object_relation': relation, - 'value': general[feature], - 'to_ids': False + "type": "text", + "object_relation": relation, + "value": general[feature], + "to_ids": False, } ) - start_time = datetime.strptime(f"{general['date']} {general['time']}", '%d/%m/%Y %H:%M:%S') + start_time = datetime.strptime(f"{general['date']} {general['time']}", "%d/%m/%Y %H:%M:%S") process_object.add_attribute( **{ - 'type': 'datetime', - 'object_relation': 'start-time', - 'value': start_time, - 'to_ids': False + "type": "datetime", + "object_relation": "start-time", + "value": start_time, + "to_ids": False, } ) self.misp_event.add_object(process_object) for field, to_call in process_activities.items(): if process.get(field): to_call(process_object.uuid, process[field]) - self.references[self.analysisinfo_uuid].append(dict(referenced_uuid=process_object.uuid, - relationship_type='calls')) - self.process_references[(general['targetid'], general['path'])] = process_object.uuid + self.references[self.analysisinfo_uuid].append( + dict(referenced_uuid=process_object.uuid, relationship_type="calls") + ) + self.process_references[(general["targetid"], general["path"])] = process_object.uuid def parse_fileactivities(self, process_uuid, fileactivities): for feature, files in fileactivities.items(): @@ -219,15 +254,15 @@ def parse_fileactivities(self, process_uuid, fileactivities): continue if files: - for call in files['call']: - self.attributes['filename'][call['path']].add((process_uuid, file_references_mapping[feature])) + for call in files["call"]: + self.attributes["filename"][call["path"]].add((process_uuid, file_references_mapping[feature])) def analysis_type(self): - generalinfo = self.data['generalinfo'] + generalinfo = self.data["generalinfo"] - if generalinfo['target']['sample']: + if generalinfo["target"]["sample"]: return "file" - elif generalinfo['target']['url']: + elif generalinfo["target"]["url"]: return "url" else: raise Exception("Unknown analysis type") @@ -239,35 +274,35 @@ def parse_url_analysis(self): self.analysisinfo_uuid = url_object.uuid url_object.add_attribute( **{ - 'type': 'url', - 'object_relation': 'url', - 'value': generalinfo["target"]["url"], - 'to_ids': False + "type": "url", + "object_relation": "url", + "value": generalinfo["target"]["url"], + "to_ids": False, } ) self.misp_event.add_object(url_object) def parse_fileinfo(self): - fileinfo = self.data['fileinfo'] + fileinfo = self.data["fileinfo"] - file_object = MISPObject('file') + file_object = MISPObject("file") self.analysisinfo_uuid = file_object.uuid for field in file_object_fields: file_object.add_attribute( **{ - 'type': field, - 'object_relation': field, - 'value': fileinfo[field], - 'to_ids': False + "type": field, + "object_relation": field, + "value": fileinfo[field], + "to_ids": False, } ) for field, mapping in file_object_mapping.items(): if fileinfo.get(field) is not None: - attribute = {'value': fileinfo[field], 'to_ids': False} + attribute = {"value": fileinfo[field], "to_ids": False} attribute.update(mapping) file_object.add_attribute(**attribute) - arch = self.data['generalinfo']['arch'] + arch = self.data["generalinfo"]["arch"] if self.import_executable and arch in arch_type_mapping: to_call = arch_type_mapping[arch] getattr(self, to_call)(fileinfo, file_object) @@ -275,269 +310,282 @@ def parse_fileinfo(self): self.misp_event.add_object(file_object) def parse_apk(self, fileinfo, file_object): - apkinfo = fileinfo['apk'] + apkinfo = fileinfo["apk"] self.misp_event.add_object(file_object) permission_lists = defaultdict(list) - for permission in apkinfo['requiredpermissions']['permission']: - permission = permission['@name'].split('.') - permission_lists[' '.join(permission[:-1])].append(permission[-1]) - attribute_type = 'text' + for permission in apkinfo["requiredpermissions"]["permission"]: + permission = permission["@name"].split(".") + permission_lists[" ".join(permission[:-1])].append(permission[-1]) + attribute_type = "text" for comment, permissions in permission_lists.items(): - permission_object = MISPObject('android-permission') + permission_object = MISPObject("android-permission") permission_object.add_attribute( **{ - 'type': attribute_type, - 'object_relation': 'comment', - 'value': comment, - 'to_ids': False + "type": attribute_type, + "object_relation": "comment", + "value": comment, + "to_ids": False, } ) for permission in permissions: permission_object.add_attribute( **{ - 'type': attribute_type, - 'object_relation': 'permission', - 'value': permission, - 'to_ids': False + "type": attribute_type, + "object_relation": "permission", + "value": permission, + "to_ids": False, } ) self.misp_event.add_object(permission_object) - self.references[file_object.uuid].append(dict(referenced_uuid=permission_object.uuid, - relationship_type='grants')) + self.references[file_object.uuid].append( + dict(referenced_uuid=permission_object.uuid, relationship_type="grants") + ) def parse_elf(self, fileinfo, file_object): - elfinfo = fileinfo['elf'] + elfinfo = fileinfo["elf"] self.misp_event.add_object(file_object) - attribute_type = 'text' - relationship = 'includes' - size = 'size-in-bytes' - for fileinfo in elfinfo['file']: - elf_object = MISPObject('elf') - self.references[file_object.uuid].append(dict(referenced_uuid=elf_object.uuid, - relationship_type=relationship)) - elf = fileinfo['main'][0]['header'][0] - if elf.get('type'): + attribute_type = "text" + relationship = "includes" + size = "size-in-bytes" + for fileinfo in elfinfo["file"]: + elf_object = MISPObject("elf") + self.references[file_object.uuid].append( + dict(referenced_uuid=elf_object.uuid, relationship_type=relationship) + ) + elf = fileinfo["main"][0]["header"][0] + if elf.get("type"): # Haven't seen anything but EXEC yet in the files I tested - attribute_value = "EXECUTABLE" if elf['type'] == "EXEC (Executable file)" else elf['type'] + attribute_value = "EXECUTABLE" if elf["type"] == "EXEC (Executable file)" else elf["type"] elf_object.add_attribute( **{ - 'type': attribute_type, - 'object_relation': 'type', - 'value': attribute_value, - 'to_ids': False + "type": attribute_type, + "object_relation": "type", + "value": attribute_value, + "to_ids": False, } ) for feature, relation in elf_object_mapping.items(): if elf.get(feature): elf_object.add_attribute( **{ - 'type': attribute_type, - 'object_relation': relation, - 'value': elf[feature], - 'to_ids': False + "type": attribute_type, + "object_relation": relation, + "value": elf[feature], + "to_ids": False, } ) - sections_number = len(fileinfo['sections']['section']) + sections_number = len(fileinfo["sections"]["section"]) elf_object.add_attribute( **{ - 'type': 'counter', - 'object_relation': 'number-sections', - 'value': sections_number, - 'to_ids': False + "type": "counter", + "object_relation": "number-sections", + "value": sections_number, + "to_ids": False, } ) self.misp_event.add_object(elf_object) - for section in fileinfo['sections']['section']: - section_object = MISPObject('elf-section') - for feature in ('name', 'type'): + for section in fileinfo["sections"]["section"]: + section_object = MISPObject("elf-section") + for feature in ("name", "type"): if section.get(feature): section_object.add_attribute( **{ - 'type': attribute_type, - 'object_relation': feature, - 'value': section[feature], - 'to_ids': False + "type": attribute_type, + "object_relation": feature, + "value": section[feature], + "to_ids": False, } ) - if section.get('size'): + if section.get("size"): section_object.add_attribute( **{ - 'type': size, - 'object_relation': size, - 'value': int(section['size'], 16), - 'to_ids': False + "type": size, + "object_relation": size, + "value": int(section["size"], 16), + "to_ids": False, } ) - for flag in section['flagsdesc']: + for flag in section["flagsdesc"]: try: attribute_value = elf_section_flags_mapping[flag] section_object.add_attribute( **{ - 'type': attribute_type, - 'object_relation': 'flag', - 'value': attribute_value, - 'to_ids': False + "type": attribute_type, + "object_relation": "flag", + "value": attribute_value, + "to_ids": False, } ) except KeyError: - print(f'Unknown elf section flag: {flag}') + print(f"Unknown elf section flag: {flag}") continue self.misp_event.add_object(section_object) - self.references[elf_object.uuid].append(dict(referenced_uuid=section_object.uuid, - relationship_type=relationship)) + self.references[elf_object.uuid].append( + dict( + referenced_uuid=section_object.uuid, + relationship_type=relationship, + ) + ) def parse_pe(self, fileinfo, file_object): try: - peinfo = fileinfo['pe'] + peinfo = fileinfo["pe"] except KeyError: self.misp_event.add_object(file_object) return - pe_object = MISPObject('pe') - relationship = 'includes' + pe_object = MISPObject("pe") + relationship = "includes" file_object.add_reference(pe_object.uuid, relationship) self.misp_event.add_object(file_object) for field, mapping in pe_object_fields.items(): if peinfo.get(field) is not None: - attribute = {'value': peinfo[field], 'to_ids': False} + attribute = {"value": peinfo[field], "to_ids": False} attribute.update(mapping) pe_object.add_attribute(**attribute) pe_object.add_attribute( **{ - 'type': 'datetime', - 'object_relation': 'compilation-timestamp', - 'value': int(peinfo['timestamp'].split()[0], 16), - 'to_ids': False + "type": "datetime", + "object_relation": "compilation-timestamp", + "value": int(peinfo["timestamp"].split()[0], 16), + "to_ids": False, } ) - program_name = fileinfo['filename'] - if peinfo['versions']: - for feature in peinfo['versions']['version']: - name = feature['name'] - if name == 'InternalName': - program_name = feature['value'] + program_name = fileinfo["filename"] + if peinfo["versions"]: + for feature in peinfo["versions"]["version"]: + name = feature["name"] + if name == "InternalName": + program_name = feature["value"] if name in pe_object_mapping: pe_object.add_attribute( **{ - 'type': 'text', - 'object_relation': pe_object_mapping[name], - 'value': feature['value'], - 'to_ids': False + "type": "text", + "object_relation": pe_object_mapping[name], + "value": feature["value"], + "to_ids": False, } ) - sections_number = len(peinfo['sections']['section']) + sections_number = len(peinfo["sections"]["section"]) pe_object.add_attribute( **{ - 'type': 'counter', - 'object_relation': 'number-sections', - 'value': sections_number, - 'to_ids': False + "type": "counter", + "object_relation": "number-sections", + "value": sections_number, + "to_ids": False, } ) - signatureinfo = peinfo['signature'] - if signatureinfo['signed']: - signerinfo_object = MISPObject('authenticode-signerinfo') - pe_object.add_reference(signerinfo_object.uuid, 'signed-by') + signatureinfo = peinfo["signature"] + if signatureinfo["signed"]: + signerinfo_object = MISPObject("authenticode-signerinfo") + pe_object.add_reference(signerinfo_object.uuid, "signed-by") self.misp_event.add_object(pe_object) signerinfo_object.add_attribute( **{ - 'type': 'text', - 'object_relation': 'program-name', - 'value': program_name, - 'to_ids': False + "type": "text", + "object_relation": "program-name", + "value": program_name, + "to_ids": False, } ) for feature, mapping in signerinfo_object_mapping.items(): if signatureinfo.get(feature) is not None: - attribute = {'value': signatureinfo[feature], 'to_ids': False} + attribute = {"value": signatureinfo[feature], "to_ids": False} attribute.update(mapping) signerinfo_object.add_attribute(**attribute) self.misp_event.add_object(signerinfo_object) else: self.misp_event.add_object(pe_object) - for section in peinfo['sections']['section']: + for section in peinfo["sections"]["section"]: section_object = self.parse_pe_section(section) - self.references[pe_object.uuid].append(dict(referenced_uuid=section_object.uuid, - relationship_type=relationship)) + self.references[pe_object.uuid].append( + dict(referenced_uuid=section_object.uuid, relationship_type=relationship) + ) self.misp_event.add_object(section_object) def parse_pe_section(self, section): - section_object = MISPObject('pe-section') + section_object = MISPObject("pe-section") for feature, mapping in pe_section_object_mapping.items(): if section.get(feature) is not None: - attribute = {'value': section[feature], 'to_ids': False} + attribute = {"value": section[feature], "to_ids": False} attribute.update(mapping) section_object.add_attribute(**attribute) return section_object def parse_network_interactions(self): - domaininfo = self.data['domaininfo'] + domaininfo = self.data["domaininfo"] if domaininfo: - for domain in domaininfo['domain']: - if domain['@ip'] != 'unknown': - domain_object = MISPObject('domain-ip') + for domain in domaininfo["domain"]: + if domain["@ip"] != "unknown": + domain_object = MISPObject("domain-ip") for key, mapping in domain_object_mapping.items(): if domain.get(key) is not None: - attribute = {'value': domain[key], 'to_ids': False} + attribute = {"value": domain[key], "to_ids": False} attribute.update(mapping) domain_object.add_attribute(**attribute) self.misp_event.add_object(domain_object) - reference = dict(referenced_uuid=domain_object.uuid, relationship_type='contacts') - self.add_process_reference(domain['@targetid'], domain['@currentpath'], reference) + reference = dict(referenced_uuid=domain_object.uuid, relationship_type="contacts") + self.add_process_reference(domain["@targetid"], domain["@currentpath"], reference) else: attribute = MISPAttribute() - attribute.from_dict(**{'type': 'domain', 'value': domain['@name'], 'to_ids': False}) + attribute.from_dict(**{"type": "domain", "value": domain["@name"], "to_ids": False}) self.misp_event.add_attribute(**attribute) - reference = dict(referenced_uuid=attribute.uuid, relationship_type='contacts') - self.add_process_reference(domain['@targetid'], domain['@currentpath'], reference) - ipinfo = self.data['ipinfo'] + reference = dict(referenced_uuid=attribute.uuid, relationship_type="contacts") + self.add_process_reference(domain["@targetid"], domain["@currentpath"], reference) + ipinfo = self.data["ipinfo"] if ipinfo: - for ip in ipinfo['ip']: + for ip in ipinfo["ip"]: attribute = MISPAttribute() - attribute.from_dict(**{'type': 'ip-dst', 'value': ip['@ip'], 'to_ids': False}) + attribute.from_dict(**{"type": "ip-dst", "value": ip["@ip"], "to_ids": False}) self.misp_event.add_attribute(**attribute) - reference = dict(referenced_uuid=attribute.uuid, relationship_type='contacts') - self.add_process_reference(ip['@targetid'], ip['@currentpath'], reference) - urlinfo = self.data['urlinfo'] + reference = dict(referenced_uuid=attribute.uuid, relationship_type="contacts") + self.add_process_reference(ip["@targetid"], ip["@currentpath"], reference) + urlinfo = self.data["urlinfo"] if urlinfo: - for url in urlinfo['url']: - target_id = int(url['@targetid']) - current_path = url['@currentpath'] + for url in urlinfo["url"]: + target_id = int(url["@targetid"]) + current_path = url["@currentpath"] attribute = MISPAttribute() - attribute_dict = {'type': 'url', 'value': url['@name'], 'to_ids': False} - if target_id != -1 and current_path != 'unknown': - self.references[self.process_references[(target_id, current_path)]].append({ - 'referenced_uuid': attribute.uuid, - 'relationship_type': 'contacts' - }) + attribute_dict = {"type": "url", "value": url["@name"], "to_ids": False} + if target_id != -1 and current_path != "unknown": + self.references[self.process_references[(target_id, current_path)]].append( + { + "referenced_uuid": attribute.uuid, + "relationship_type": "contacts", + } + ) else: - attribute_dict['comment'] = 'From Memory - Enriched via the joe_import module' + attribute_dict["comment"] = "From Memory - Enriched via the joe_import module" attribute.from_dict(**attribute_dict) self.misp_event.add_attribute(**attribute) def parse_registryactivities(self, process_uuid, registryactivities): - if registryactivities['keyCreated']: - for call in registryactivities['keyCreated']['call']: - self.attributes['regkey'][call['path']].add((process_uuid, 'creates')) + if registryactivities["keyCreated"]: + for call in registryactivities["keyCreated"]["call"]: + self.attributes["regkey"][call["path"]].add((process_uuid, "creates")) for feature, relationship in registry_references_mapping.items(): if registryactivities[feature]: - for call in registryactivities[feature]['call']: - registry_key = MISPObject('registry-key') + for call in registryactivities[feature]["call"]: + registry_key = MISPObject("registry-key") for field, mapping in regkey_object_mapping.items(): if call.get(field) is not None: - attribute = {'value': call[field], 'to_ids': False} + attribute = {"value": call[field], "to_ids": False} attribute.update(mapping) registry_key.add_attribute(**attribute) registry_key.add_attribute( **{ - 'type': 'text', - 'object_relation': 'data-type', - 'value': f"REG_{call['type'].upper()}", - 'to_ids': False + "type": "text", + "object_relation": "data-type", + "value": f"REG_{call['type'].upper()}", + "to_ids": False, } ) self.misp_event.add_object(registry_key) - self.references[process_uuid].append(dict(referenced_uuid=registry_key.uuid, - relationship_type=relationship)) + self.references[process_uuid].append( + dict( + referenced_uuid=registry_key.uuid, + relationship_type=relationship, + ) + ) def add_process_reference(self, target, currentpath, reference): try: @@ -547,7 +595,7 @@ def add_process_reference(self, target, currentpath, reference): def create_attribute(self, attribute_type, attribute_value): attribute = MISPAttribute() - attribute.from_dict(**{'type': attribute_type, 'value': attribute_value, 'to_ids': False}) + attribute.from_dict(**{"type": attribute_type, "value": attribute_value, "to_ids": False}) self.misp_event.add_attribute(**attribute) return attribute.uuid @@ -555,19 +603,19 @@ def finalize_results(self): if self.references: self.build_references() event = json.loads(self.misp_event.to_json()) - self.results = {key: event[key] for key in ('Attribute', 'Object', 'Tag') if (key in event and event[key])} + self.results = {key: event[key] for key in ("Attribute", "Object", "Tag") if (key in event and event[key])} @staticmethod def parse_timestamp(timestamp): - timestamp = timestamp.split(':') - timestamp[-1] = str(round(float(timestamp[-1].split(' ')[0]), 6)) - return ':'.join(timestamp) + timestamp = timestamp.split(":") + timestamp[-1] = str(round(float(timestamp[-1].split(" ")[0]), 6)) + return ":".join(timestamp) @staticmethod def prefetch_attributes_data(connection): attributes = [] for field, value in zip(network_behavior_fields, connection): - attribute = {'value': value, 'to_ids': False} + attribute = {"value": value, "to_ids": False} attribute.update(network_connection_object_mapping[field]) attributes.append(attribute) return attributes diff --git a/misp_modules/lib/lastline_api.py b/misp_modules/lib/lastline_api.py index 83726adb8..619762bce 100644 --- a/misp_modules/lib/lastline_api.py +++ b/misp_modules/lib/lastline_api.py @@ -30,24 +30,27 @@ ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. """ + import abc -import logging import io import ipaddress -import pymisp +import logging import re -import requests from urllib import parse +import pymisp +import requests DEFAULT_LL_PORTAL_API_URL = "https://user.lastline.com/papi" DEFAULT_LL_ANALYSIS_API_URL = "https://analysis.lastline.com" -LL_HOSTED_DOMAINS = frozenset([ - "user.lastline.com", - "user.emea.lastline.com", -]) +LL_HOSTED_DOMAINS = frozenset( + [ + "user.lastline.com", + "user.emea.lastline.com", + ] +) def purge_none(d): @@ -129,6 +132,7 @@ class Error(Exception): class ApiError(Error): """Server error with a message and an error code.""" + def __init__(self, error_msg, error_code=None): super(ApiError, self).__init__(error_msg, error_code) self.error_msg = error_msg @@ -143,11 +147,11 @@ def __str__(self): class LastlineAbstractClient(abc.ABC): - """"A very basic HTTP client providing basic functionality.""" + """ "A very basic HTTP client providing basic functionality.""" __metaclass__ = abc.ABCMeta - SUB_APIS = ('analysis', 'authentication', 'knowledgebase', 'login') + SUB_APIS = ("analysis", "authentication", "knowledgebase", "login") FORMATS = ["json", "xml"] @classmethod @@ -348,7 +352,7 @@ def do_request( raw=False, raw_response=False, headers=None, - stream_response=False + stream_response=False, ): if raw_response: raw = True @@ -420,8 +424,8 @@ def get_progress(self, uuid): "progress": 100 } """ - url = self._build_url('analysis', ['get_progress']) - params = {'uuid': uuid} + url = self._build_url("analysis", ["get_progress"]) + params = {"uuid": uuid} return self.do_request("POST", url, params=params) def get_result(self, uuid): @@ -438,8 +442,8 @@ def get_result(self, uuid): """ # better: use 'get_results()' but that would break # backwards-compatibility - url = self._build_url('analysis', ['get']) - params = {'uuid': uuid} + url = self._build_url("analysis", ["get"]) + params = {"uuid": uuid} return self.do_request("GET", url, params=params) def submit_file( @@ -486,27 +490,31 @@ def submit_file( """ file_stream = io.BytesIO(file_data) api_url = self._build_url("analysis", ["submit", "file"]) - params = purge_none({ - "bypass_cache": bypass_cache and 1 or None, - "analysis_timeout": analysis_timeout, - "analysis_env": analysis_env, - "allow_network_traffic": allow_network_traffic and 1 or None, - "filename": file_name, - "password": password, - "full_report_score": -1, - }) - - files = purge_none({ - # If an explicit filename was provided, we can pass it down to - # python-requests to use it in the multipart/form-data. This avoids - # having python-requests trying to guess the filename based on stream - # attributes. - # - # The problem with this is that, if the filename is not ASCII, then - # this triggers a bug in flask/werkzeug which means the file is - # thrown away. Thus, we just force an ASCII name - "file": ('dummy-ascii-name-for-file-param', file_stream), - }) + params = purge_none( + { + "bypass_cache": bypass_cache and 1 or None, + "analysis_timeout": analysis_timeout, + "analysis_env": analysis_env, + "allow_network_traffic": allow_network_traffic and 1 or None, + "filename": file_name, + "password": password, + "full_report_score": -1, + } + ) + + files = purge_none( + { + # If an explicit filename was provided, we can pass it down to + # python-requests to use it in the multipart/form-data. This avoids + # having python-requests trying to guess the filename based on stream + # attributes. + # + # The problem with this is that, if the filename is not ASCII, then + # this triggers a bug in flask/werkzeug which means the file is + # thrown away. Thus, we just force an ASCII name + "file": ("dummy-ascii-name-for-file-param", file_stream), + } + ) return self.do_request("POST", api_url, params=params, files=files) @@ -547,12 +555,14 @@ def submit_url( } """ api_url = self._build_url("analysis", ["submit", "url"]) - params = purge_none({ - "url": url, - "referer": referer, - "bypass_cache": bypass_cache and 1 or None, - "user_agent": user_agent or None, - }) + params = purge_none( + { + "url": url, + "referer": referer, + "bypass_cache": bypass_cache and 1 or None, + "user_agent": user_agent or None, + } + ) return self.do_request("POST", api_url, params=params) @@ -647,7 +657,7 @@ def submit_url( "url": url, "bypass_cache": bypass_cache, "referer": referer, - "user_agent": user_agent + "user_agent": user_agent, } ) return self.post("analysis", "submit_url", params=params) @@ -724,12 +734,16 @@ def __init__(self): @staticmethod def _get_mitre_techniques(result): return [ - "misp-galaxy:mitre-attack-pattern=\"{} - {}\"".format(w[0], w[1]) - for w in sorted(set([ - (y["id"], y["name"]) - for x in result.get("malicious_activity", []) - for y in result.get("activity_to_mitre_techniques", {}).get(x, []) - ])) + 'misp-galaxy:mitre-attack-pattern="{} - {}"'.format(w[0], w[1]) + for w in sorted( + set( + [ + (y["id"], y["name"]) + for x in result.get("malicious_activity", []) + for y in result.get("activity_to_mitre_techniques", {}).get(x, []) + ] + ) + ) ] def parse(self, analysis_link, result): @@ -755,7 +769,7 @@ def parse(self, analysis_link, result): o.add_attribute( "mimetype", type="mime-type", - value=result["analysis_subject"]["mime_type"] + value=result["analysis_subject"]["mime_type"], ) self.misp_event.add_object(o) @@ -763,10 +777,10 @@ def parse(self, analysis_link, result): network_dict = result.get("report", {}).get("analysis", {}).get("network", {}) for request in network_dict.get("requests", []): parsed_uri = parse.urlparse(request["url"]) - o = pymisp.MISPObject(name='http-request') - o.add_attribute('host', parsed_uri.netloc) - o.add_attribute('method', "GET") - o.add_attribute('uri', request["url"]) + o = pymisp.MISPObject(name="http-request") + o.add_attribute("host", parsed_uri.netloc) + o.add_attribute("method", "GET") + o.add_attribute("uri", request["url"]) o.add_attribute("ip", request["ip"]) self.misp_event.add_object(o) @@ -785,8 +799,8 @@ def parse(self, analysis_link, result): except ValueError: pass - o = pymisp.MISPObject(name='dns-record') - o.add_attribute('queried-domain', hostname) + o = pymisp.MISPObject(name="dns-record") + o.add_attribute("queried-domain", hostname) self.misp_event.add_object(o) # Add HTTP conversations (as network connection and as http request) @@ -809,13 +823,13 @@ def parse(self, analysis_link, result): uri = "http://{}:{}{}".format( http_conversation["dst_host"], http_conversation["dst_port"], - path + path, ) - o = pymisp.MISPObject(name='http-request') - o.add_attribute('host', http_conversation["dst_host"]) - o.add_attribute('method', method) - o.add_attribute('uri', uri) - o.add_attribute('ip', http_conversation["dst_ip"]) + o = pymisp.MISPObject(name="http-request") + o.add_attribute("host", http_conversation["dst_host"]) + o.add_attribute("method", method) + o.add_attribute("uri", uri) + o.add_attribute("ip", http_conversation["dst_ip"]) self.misp_event.add_object(o) # Add sandbox info like score and sandbox type diff --git a/misp_modules/lib/onyphe/LICENSE b/misp_modules/lib/onyphe/LICENSE new file mode 100644 index 000000000..425fb40ec --- /dev/null +++ b/misp_modules/lib/onyphe/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 Sebastien Larinier + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/misp_modules/lib/onyphe/__init__.py b/misp_modules/lib/onyphe/__init__.py new file mode 100644 index 000000000..5e35bab4b --- /dev/null +++ b/misp_modules/lib/onyphe/__init__.py @@ -0,0 +1,2 @@ +from onyphe.client import Onyphe +from onyphe.exception import APIError diff --git a/misp_modules/lib/onyphe/client.py b/misp_modules/lib/onyphe/client.py new file mode 100644 index 000000000..393fe3a00 --- /dev/null +++ b/misp_modules/lib/onyphe/client.py @@ -0,0 +1,701 @@ +import json +import logging +import os +from urllib.parse import urljoin + +from onyphe.exception import APIError, ParamError + +""" +onyphe.client +~~~~~~~~~~~~~ + +This module implements the Onyphe API. + +:copyright: (c) 2017- by Sebastien Larinier +""" +import requests +from requests.utils import quote + + +class Onyphe: + """Wrapper around the Onyphe REST + + :param key: The Onyphe API key that can be obtained from your account page (https://www.onyphe.io) + :type key: str + """ + + def __init__(self, api_key, version="v2"): + self.api_key = api_key + self.base_url = "https://www.onyphe.io/api/" + self.version = version + self._session = requests + + self.methods = { + "get": self._session.get, + "post": self._session.post, + } + + def _choose_url(self, uri): + self.url = urljoin(self.base_url, uri) + + def _request(self, method, payload, json_data, files): + + data = None + session = self.methods[method] + try: + if json_data: + response = session(self.url, params=payload, data=json.dumps(json_data)) + elif files: + payload["Content-Type"] = "application/json" + response = session(self.url, params=payload, data=files) + else: + response = self.methods[method](self.url, params=payload) + except: + raise APIError("Unable to connect to Onyphe") + + if response.status_code == requests.codes.NOT_FOUND: + + raise APIError("Page Not found %s" % self.url) + elif response.status_code == requests.codes.FORBIDDEN: + raise APIError("Access Forbidden") + elif response.status_code == requests.codes.too_many_requests: + raise APIError("Too Many Requests") + elif response.status_code != requests.codes.OK: + try: + error = response.json()["text"] + except Exception as e: + error = "Unknown error" + + raise APIError(error) + try: + data = response.json() + return data + + except: + return response + + def _prepare_request(self, uri, **kwargs): + params = {"apikey": self.api_key} + + json_data = None + + files = None + + if "page" in kwargs: + params["page"] = kwargs["page"] + + if "json_data" in kwargs: + json_data = kwargs["json_data"] + if "files" in kwargs: + files = kwargs["files"] + + method = kwargs["method"] + + self._choose_url(uri) + + data = self._request(method, params, json_data, files) + if data: + return data + + def __search(self, query, endpoint, **kwargs): + return self._prepare_request(quote("/".join([self.version, "search", query])), **kwargs) + + def synscan(self, ip): + """Call API Onyphe https://www.onyphe.io/api/v1/synscan/ + + :param ip: IPv4 or IPv6 address + :type ip: str + :returns: dict -- a dictionary containing the results of the search about synscans. + """ + return self._prepare_request("/".join([self.version, "synscan", ip]), method="get") + + def summary_ip(self, ip): + """Call API Onyphe https://www.onyphe.io/api/v2/summary/ip/ + + :param ip: IPv4 or IPv6 address + :type ip: str + :returns: dict -- a dictionary containing all informations of IP + """ + return self._prepare_request("/".join([self.version, "summary/ip", ip]), method="get") + + def summary_domain(self, domain): + """Call API Onyphe https://www.onyphe.io/api/v2/summary/domain/ + + :param domain: domain + :type domain: str + :returns: dict -- a dictionary containing the results of the summary of domain. + """ + return self._prepare_request("/".join([self.version, "summary/domain", domain]), method="get") + + def summary_hostname(self, hostname): + """Call API Onyphe https://www.onyphe.io/api/v2/summary/hostname/ + + :param hostname: hostname + :type hostname: str + :returns: dict -- a dictionary containing the results of the summary of hostname. + """ + return self._prepare_request("/".join([self.version, "summary/hostname", hostname]), method="get") + + def simple_geoloc(self, ip): + """Call API Onyphe https://www.onyphe.io/api/v2/simple/geoloc/ + + :param ip: IPv4 or IPv6 address + :type ip: str + :returns: dict -- a dictionary containing the results of synscan of IP + """ + return self._prepare_request("/".join([self.version, "simple/geoloc", ip]), method="get") + + def simple_geoloc_best(self, ip): + """Call API Onyphe https://www.onyphe.io/api/v2/simple/geoloc/best + + :param ip: IPv4 or IPv6 address + :type ip: str + :returns: dict -- a dictionary containing the results of geoloc of IP + """ + return self._prepare_request("/".join([self.version, "simple/geoloc/best", ip]), method="get") + + def simple_inetnum(self, ip): + """Call API Onyphe https://www.onyphe.io/api/v2/simple/inetnum/ + + :param ip: IPv4 or IPv6 address + :type ip: str + :returns: dict -- a dictionary containing the results of inetnum of IP + """ + return self._prepare_request("/".join([self.version, "simple/inetnum", ip]), method="get") + + def simple_inetnum_best(self, ip): + """Call API Onyphe https://www.onyphe.io/api/v2/simple/inetnum/best + + :param ip: IPv4 or IPv6 address + :type ip: str + :returns: dict -- a dictionary containing the results of intenum of IP + """ + return self._prepare_request("/".join([self.version, "simple/inetnum/best", ip]), method="get") + + def simple_threatlist_best(self, ip): + """Call API Onyphe https://www.onyphe.io/api/v2/simple/threatlist/best + + :param ip: IPv4 or IPv6 address + :type ip: str + :returns: dict -- a dictionary containing the results of threatlist with best API of IP + """ + return self._prepare_request("/".join([self.version, "simple/threatlist/best", ip]), method="get") + + def simple_pastries(self, ip): + """Call API Onyphe https://www.onyphe.io/api/v2/simple/pastries/ + + :param ip: IPv4 or IPv6 address + :type ip: str + :returns: dict -- a dictionary containing the results of pastries of IP + """ + return self._prepare_request("/".join([self.version, "simple/pastries", ip]), method="get") + + def simple_resolver(self, ip): + """Call API Onyphe https://www.onyphe.io/api/v2/simple/resolver/ + + :param ip: IPv4 or IPv6 address + :type ip: str + :returns: dict -- a dictionary containing the results of resolver of IP + """ + return self._prepare_request("/".join([self.version, "simple/resolver", ip]), method="get") + + def simple_sniffer(self, ip): + """Call API Onyphe https://www.onyphe.io/api/v2/simple/sniffer/ + + :param ip: IPv4 or IPv6 address + :type ip: str + :returns: dict -- a dictionary containing the results of sniffer of IP + """ + return self._prepare_request("/".join([self.version, "simple/sniffer", ip]), method="get") + + def simple_synscan(self, ip): + """Call API Onyphe https://www.onyphe.io/api/v2/simple/synscan/ + + :param ip: IPv4 or IPv6 address + :type ip: str + :returns: dict -- a dictionary containing the results of synscan of IP + """ + return self._prepare_request("/".join([self.version, "simple/synscan", ip]), method="get") + + def simple_threatlist(self, ip): + """Call API Onyphe https://www.onyphe.io/api/v2/simple/threatlist/ + + :param ip: IPv4 or IPv6 address + :type ip: str + :returns: dict -- a dictionary containing the results of threatlist of IP + """ + return self._prepare_request("/".join([self.version, "simple/threatlist", ip]), method="get") + + def simple_topsite(self, ip): + """Call API Onyphe https://www.onyphe.io/api/v2/simple/topsite/ + + :param ip: IPv4 or IPv6 address + :type ip: str + :returns: dict -- a dictionary containing the results of topsite of IP + """ + return self._prepare_request("/".join([self.version, "simple/topsite", ip]), method="get") + + def simple_vulnscan(self, ip): + """Call API Onyphe https://www.onyphe.io/api/v2/simple/vulnscan/ + + :param ip: IPv4 or IPv6 address + :type ip: str + :returns: dict -- a dictionary containing the results of vulnscan of IP + """ + return self._prepare_request("/".join([self.version, "simple/vulnscan", ip]), method="get") + + def simple_onionshot(self, ip): + """Call API Onyphe https://www.onyphe.io/api/v2/simple/onionshot/ + + :param ip: IPv4 or IPv6 address + :type ip: str + :returns: dict -- a dictionary containing the results of onionshot of IP + """ + return self._prepare_request("/".join([self.version, "simple/onionshot", ip]), method="get") + + def simple_datashot(self, ip): + """Call API Onyphe https://www.onyphe.io/api/v2/simple/datashot/ + + :param ip: IPv4 or IPv6 address + :type ip: str + :returns: dict -- a dictionary containing the results of datashot of IP + """ + return self._prepare_request("/".join([self.version, "simple/datashot", ip]), method="get") + + def simple_ctl(self, data): + """Call API Onyphe https://www.onyphe.io/api/v2/ctl/{,,, + + :param data_md5: category of information we have for the given domain or hostname + :type data_md5: str + :returns: dict -- a dictionary containing Information onionscan on domain or hostname + """ + return self._prepare_request("/".join([self.version, "simple/datascan/datamd5", data_md5]), method="get") + + def simple_resolver_forward(self, ip): + """Call API Onyphe https://www.onyphe.io/api/v2/resolver/forward/ + :param ip: IPv4 or IPv6 address + :type ip: str + :returns: dict -- a dictionary containing the results of forward of IP + """ + return self.__resolver(ip, "forward") + + def simple_resolver_reverse(self, ip): + """Call API Onyphe https://www.onyphe.io/api/v2/resolver/reverse/ + + :param ip: IPv4 or IPv6 address + :type ip: str + :returns: dict -- a dictionary containing the results of reverse of IP + """ + return self.__resolver(ip, "reverse") + + def __resolver(self, ip, type_resolv): + return self._prepare_request( + "/".join([self.version, "simple/resolver/%s" % type_resolv, ip]), + method="get", + ) + + def search(self, query, **kwargs): + """Call API Onyphe https://www.onyphe.io/api/v2/search/ + :param query: example product:Apache port:443 os:Windows. + :type query: str + :return: dict -- a dictionary with result + """ + kwargs["method"] = "get" + return self.__search(query, "datascan", **kwargs) + + def alert_list(self): + """Call API Onyphe https://www.onyphe.io/api/v2/alert/list + + :return: dict -- a dictionary with result + """ + return self._prepare_request("/".join([self.version, "alert/list"]), method="get") + + def add_alert(self, query, name, email): + """Call API Onyphe https://www.onyphe.io/api/v2/alert/add + :param query: query language onyphe + :type query: str + :param name: name of alert + :type name: str + :param email: email to receive alerts + :type email: str + :return: dict -- a dictionary with result + """ + if query and name and email: + data = {"query": query, "name": name, "email": email} + + return self._prepare_request("/".join([self.version, "alert/add"]), method="post", json_data=data) + else: + raise ParamError("Parameters Invalid") + + def del_alert(self, id_alert): + """Call API Onyphe https://www.onyphe.io/api/v2/alert/del + + :param id_alert: id of alert to delete + :type id_alert: str + :return: dict -- a dictionary with result + """ + if id_alert: + return self._prepare_request("/".join([self.version, "alert/del", id_alert]), method="post") + else: + raise ParamError("Parameter Invalid") + + def bulk_summary_ip(self, path): + """Call API Onyphe https://www.onyphe.io/api/v2/bulk/summary/ip + + :param path: path of the files with IPs + :type path:str + :return: dict -- a dictionary with result + """ + if os.path.isfile(path): + + file_iocs = open(path, "rb") + for line in self._prepare_request( + "/".join([self.version, "bulk/summary/ip"]), + method="post", + files=file_iocs, + ).iter_lines(): + yield json.loads(line.decode("utf-8")) + + else: + raise ParamError("%s is no a file" % path) + + def bulk_summary_domain(self, path): + """Call API Onyphe https://www.onyphe.io/api/v2/bulk/summary/domain + + :param path: path of the files with domains + :type path:str + :return: dict -- a dictionary with result + """ + if os.path.isfile(path): + + file_iocs = open(path, "rb") + for line in self._prepare_request( + "/".join([self.version, "bulk/summary/domain"]), + method="post", + files=file_iocs, + ).iter_lines(): + yield json.loads(line.decode("utf-8")) + + else: + raise ParamError("%s is no a file" % path) + + def bulk_summary_hostname(self, path): + """Call API Onyphe https://www.onyphe.io/api/v2/bulk/summary/hostname + + :param path: path of the files with hostnames + :type path:str + :return: dict -- a dictionary with result + """ + if os.path.isfile(path): + + file_iocs = open(path, "rb") + for line in self._prepare_request( + "/".join([self.version, "bulk/summary/hostname"]), + method="post", + files=file_iocs, + ).iter_lines(): + yield json.loads(line.decode("utf-8")) + + else: + raise ParamError("%s is no a file" % path) + + def bulk_simple_ctl_ip(self, path): + """Call API Onyphe https://www.onyphe.io/api/v2/bulk/simple/ctl/ip + + :param path: path of the files with IPs + :type path:str + :return: dict -- a dictionary with result + """ + if os.path.isfile(path): + + file_iocs = open(path, "rb") + for line in self._prepare_request( + "/".join([self.version, "bulk/simple/clt/ip"]), + method="post", + files=file_iocs, + ).iter_lines(): + yield json.loads(line.decode("utf-8")) + + else: + raise ParamError("%s is no a file" % path) + + def bulk_simple_datascan_ip(self, path): + """Call API Onyphe https://www.onyphe.io/api/v2/bulk/simple/datascan/ip + + :param path: path of the files with IPs + :type path:str + :return: dict -- a dictionary with result + """ + if os.path.isfile(path): + + file_iocs = open(path, "rb") + for line in self._prepare_request( + "/".join([self.version, "bulk/simple/datascan/ip"]), + method="post", + files=file_iocs, + ).iter_lines(): + yield json.loads(line.decode("utf-8")) + + else: + raise ParamError("%s is no a file" % path) + + def bulk_simple_datashot_ip(self, path): + """Call API Onyphe https://www.onyphe.io/api/v2/bulk/simple/datashot/ip + + :param path: path of the files with IPs + :type path:str + :return: dict -- a dictionary with result + """ + if os.path.isfile(path): + + file_iocs = open(path, "rb") + for line in self._prepare_request( + "/".join([self.version, "bulk/simple/datashot/ip"]), + method="post", + files=file_iocs, + ).iter_lines(): + yield json.loads(line.decode("utf-8")) + + else: + raise ParamError("%s is no a file" % path) + + def bulk_simple_geoloc_ip(self, path): + """Call API Onyphe https://www.onyphe.io/api/v2/bulk/simple/geoloc/ip + + :param path: path of the files with IPs + :type path:str + :return: dict -- a dictionary with result + """ + if os.path.isfile(path): + + file_iocs = open(path, "rb") + for line in self._prepare_request( + "/".join([self.version, "bulk/simple/geoloc/ip"]), + method="post", + files=file_iocs, + ).iter_lines(): + yield json.loads(line.decode("utf-8")) + + else: + raise ParamError("%s is no a file" % path) + + def bulk_simple_inetnum_ip(self, path): + """Call API Onyphe https://www.onyphe.io/api/v2/bulk/simple/inetnum/ip + + :param path: path of the files with IPs + :type path:str + :return: dict -- a dictionary with result + """ + if os.path.isfile(path): + + file_iocs = open(path, "rb") + for line in self._prepare_request( + "/".join([self.version, "bulk/simple/inetenum/ip"]), + method="post", + files=file_iocs, + ).iter_lines(): + yield json.loads(line.decode("utf-8")) + + else: + raise ParamError("%s is no a file" % path) + + def bulk_simple_pastries_ip(self, path): + """Call API Onyphe https://www.onyphe.io/api/v2/bulk/simple/pastries/ip + + :param path: path of the files with IPs + :type path:str + :return: dict -- a dictionary with result + """ + if os.path.isfile(path): + + file_iocs = open(path, "rb") + for line in self._prepare_request( + "/".join([self.version, "bulk/simple/pastries/ip"]), + method="post", + files=file_iocs, + ).iter_lines(): + yield json.loads(line.decode("utf-8")) + + else: + raise ParamError("%s is no a file" % path) + + def bulk_simple_resolver_ip(self, path): + """Call API Onyphe https://www.onyphe.io/api/v2/bulk/simple/resolver/ip + + :param path: path of the files with IPs + :type path:str + :return: dict -- a dictionary with result + """ + if os.path.isfile(path): + + file_iocs = open(path, "rb") + for line in self._prepare_request( + "/".join([self.version, "bulk/simple/resolver/ip"]), + method="post", + files=file_iocs, + ).iter_lines(): + yield json.loads(line.decode("utf-8")) + + else: + raise ParamError("%s is no a file" % path) + + def bulk_simple_sniffer_ip(self, path): + """Call API Onyphe https://www.onyphe.io/api/v2/bulk/simple/sniffer/ip + + :param path: path of the files with IPs + :type path:str + :return: dict -- a dictionary with result + """ + if os.path.isfile(path): + + file_iocs = open(path, "rb") + for line in self._prepare_request( + "/".join([self.version, "bulk/simple/sniffer/ip"]), + method="post", + files=file_iocs, + ).iter_lines(): + yield json.loads(line.decode("utf-8")) + + else: + raise ParamError("%s is no a file" % path) + + def bulk_simple_synscan_ip(self, path): + """Call API Onyphe https://www.onyphe.io/api/v2/bulk/simple/synscan/ip + + :param path: path of the files with IPs + :type path:str + :return: dict -- a dictionary with result + """ + if os.path.isfile(path): + + file_iocs = open(path, "rb") + for line in self._prepare_request( + "/".join([self.version, "bulk/simple/synscan/ip"]), + method="post", + files=file_iocs, + ).iter_lines(): + yield json.loads(line.decode("utf-8")) + + else: + raise ParamError("%s is no a file" % path) + + def bulk_simple_threatlist_ip(self, path): + """Call API Onyphe https://www.onyphe.io/api/v2/bulk/simple/threatlist/ip + + :param path: path of the files with IPs + :type path:str + :return: dict -- a dictionary with result + """ + if os.path.isfile(path): + + file_iocs = open(path, "rb") + for line in self._prepare_request( + "/".join([self.version, "bulk/simple/threatlist/ip"]), + method="post", + files=file_iocs, + ).iter_lines(): + yield json.loads(line.decode("utf-8")) + + else: + raise ParamError("%s is no a file" % path) + + def bulk_simple_topsite_ip(self, path): + """Call API Onyphe https://www.onyphe.io/api/v2/bulk/simple/topsite/ip + + :param path: path of the files with IPs + :type path:str + :return: dict -- a dictionary with result + """ + if os.path.isfile(path): + + file_iocs = open(path, "rb") + for line in self._prepare_request( + "/".join([self.version, "bulk/simple/topsite/ip"]), + method="post", + files=file_iocs, + ).iter_lines(): + yield json.loads(line.decode("utf-8")) + + else: + raise ParamError("%s is no a file" % path) + + def bulk_simple_vulnscan_ip(self, path): + """Call API Onyphe https://www.onyphe.io/api/v2/bulk/simple/vulnscan/ip + + :param path: path of the files with IPs + :type path:str + :return: dict -- a dictionary with result + """ + if os.path.isfile(path): + + file_iocs = open(path, "rb") + for line in self._prepare_request( + "/".join([self.version, "bulk/simple/vulnscan/ip"]), + method="post", + files=file_iocs, + ).iter_lines(): + yield json.loads(line.decode("utf-8")) + + else: + raise ParamError("%s is no a file" % path) + + def bulk_simple_whois_ip(self, path): + """Call API Onyphe https://www.onyphe.io/api/v2/bulk/simple/whois/ip + + :param path: path of the files with IPs + :type path:str + :return: dict -- a dictionary with result + """ + if os.path.isfile(path): + + file_iocs = open(path, "rb") + for line in self._prepare_request( + "/".join([self.version, "bulk/simple/whois/ip"]), + method="post", + files=file_iocs, + ).iter_lines(): + yield json.loads(line.decode("utf-8")) + + else: + raise ParamError("%s is no a file" % path) + + def export(self, query): + """Call API Onyphe https://www.onyphe.io/api/v2/export/ + :param query: example: category:datascan product:Nginx protocol:http os:Windows tls:true + :type query:str + :return: dict -- a dictionary with result + """ + uri = quote("/".join([self.version, "export", query])) + params = {"apikey": self.api_key} + self._choose_url(uri) + s = requests.Session() + with s.get(self.url, params=params, stream=True) as r: + for line in r.iter_lines(): + yield line diff --git a/misp_modules/lib/onyphe/exception.py b/misp_modules/lib/onyphe/exception.py new file mode 100644 index 000000000..a59d2dc25 --- /dev/null +++ b/misp_modules/lib/onyphe/exception.py @@ -0,0 +1,16 @@ +class APIError(Exception): + """This exception gets raised whenever a non-200 status code was returned by the Onyphe API.""" + + def __init__(self, value): + self.value = value + + def __str__(self): + return self.value + + +class ParamError(Exception): + def __init__(self, value): + self.value = value + + def __str__(self): + return self.value diff --git a/misp_modules/lib/qintel_helper.py b/misp_modules/lib/qintel_helper.py index 47106f78a..864124332 100644 --- a/misp_modules/lib/qintel_helper.py +++ b/misp_modules/lib/qintel_helper.py @@ -1,52 +1,44 @@ # Copyright (c) 2009-2021 Qintel, LLC # Licensed under Apache 2.0 (https://www.apache.org/licenses/LICENSE-2.0.txt) -from urllib.request import Request, urlopen -from urllib.parse import urlencode -from urllib.error import HTTPError -from time import sleep -from json import loads import os from copy import deepcopy from datetime import datetime, timedelta from gzip import GzipFile +from json import loads +from time import sleep +from urllib.error import HTTPError +from urllib.parse import urlencode +from urllib.request import Request, urlopen -VERSION = '1.0.1' -USER_AGENT = 'integrations-helper' +VERSION = "1.0.1" +USER_AGENT = "integrations-helper" MAX_RETRY_ATTEMPTS = 5 -DEFAULT_HEADERS = { - 'User-Agent': f'{USER_AGENT}/{VERSION}' -} +DEFAULT_HEADERS = {"User-Agent": f"{USER_AGENT}/{VERSION}"} REMOTE_MAP = { - 'pmi': 'https://api.pmi.qintel.com', - 'qwatch': 'https://api.qwatch.qintel.com', - 'qauth': 'https://api.qauth.qintel.com', - 'qsentry_feed': 'https://qsentry.qintel.com', - 'qsentry': 'https://api.qsentry.qintel.com' + "pmi": "https://api.pmi.qintel.com", + "qwatch": "https://api.qwatch.qintel.com", + "qauth": "https://api.qauth.qintel.com", + "qsentry_feed": "https://qsentry.qintel.com", + "qsentry": "https://api.qsentry.qintel.com", } ENDPOINT_MAP = { - 'pmi': { - 'ping': '/users/me', - 'cve': 'cves' - }, - 'qsentry_feed': { - 'anon': '/files/anonymization', - 'mal_hosting': '/files/malicious_hosting' - }, - 'qsentry': {}, - 'qwatch': { - 'ping': '/users/me', - 'exposures': 'exposures' + "pmi": {"ping": "/users/me", "cve": "cves"}, + "qsentry_feed": { + "anon": "/files/anonymization", + "mal_hosting": "/files/malicious_hosting", }, - 'qauth': {} + "qsentry": {}, + "qwatch": {"ping": "/users/me", "exposures": "exposures"}, + "qauth": {}, } def _get_request_wait_time(attempts): - """ Use Fibonacci numbers for determining the time to wait when rate limits + """Use Fibonacci numbers for determining the time to wait when rate limits have been encountered. """ @@ -59,12 +51,12 @@ def _get_request_wait_time(attempts): def _search(**kwargs): - remote = kwargs.get('remote') - max_retries = int(kwargs.get('max_retries', MAX_RETRY_ATTEMPTS)) - params = kwargs.get('params', {}) + remote = kwargs.get("remote") + max_retries = int(kwargs.get("max_retries", MAX_RETRY_ATTEMPTS)) + params = kwargs.get("params", {}) headers = _set_headers(**kwargs) - logger = kwargs.get('logger') + logger = kwargs.get("logger") params = urlencode(params) url = remote + "?" + params @@ -79,30 +71,29 @@ def _search(**kwargs): response = e except Exception as e: - raise Exception('API connection error') from e + raise Exception("API connection error") from e if response.code not in [429, 504]: - raise Exception(f'API connection error: {response}') + raise Exception(f"API connection error: {response}") if request_attempts < max_retries: wait_time = _get_request_wait_time(request_attempts) if response.code == 429: - msg = 'rate limit reached on attempt {request_attempts}, ' \ - 'waiting {wait_time} seconds' + msg = "rate limit reached on attempt {request_attempts}, waiting {wait_time} seconds" if logger: logger(msg) else: - msg = f'connection timed out, retrying in {wait_time} seconds' + msg = f"connection timed out, retrying in {wait_time} seconds" if logger: logger(msg) sleep(wait_time) else: - raise Exception('Max API retries exceeded') + raise Exception("Max API retries exceeded") request_attempts += 1 @@ -110,42 +101,41 @@ def _search(**kwargs): def _set_headers(**kwargs): headers = deepcopy(DEFAULT_HEADERS) - if kwargs.get('user_agent'): - headers['User-Agent'] = \ - f"{kwargs['user_agent']}/{USER_AGENT}/{VERSION}" + if kwargs.get("user_agent"): + headers["User-Agent"] = f"{kwargs['user_agent']}/{USER_AGENT}/{VERSION}" # TODO: deprecate - if kwargs.get('client_id') or kwargs.get('client_secret'): + if kwargs.get("client_id") or kwargs.get("client_secret"): try: - headers['Cf-Access-Client-Id'] = kwargs['client_id'] - headers['Cf-Access-Client-Secret'] = kwargs['client_secret'] + headers["Cf-Access-Client-Id"] = kwargs["client_id"] + headers["Cf-Access-Client-Secret"] = kwargs["client_secret"] except KeyError: - raise Exception('missing client_id or client_secret') + raise Exception("missing client_id or client_secret") - if kwargs.get('token'): - headers['x-api-key'] = kwargs['token'] + if kwargs.get("token"): + headers["x-api-key"] = kwargs["token"] return headers def _set_remote(product, query_type, **kwargs): - remote = kwargs.get('remote') - endpoint = kwargs.get('endpoint', ENDPOINT_MAP[product].get(query_type)) + remote = kwargs.get("remote") + endpoint = kwargs.get("endpoint", ENDPOINT_MAP[product].get(query_type)) if not remote: remote = REMOTE_MAP[product] if not endpoint: - raise Exception('invalid search type') + raise Exception("invalid search type") - remote = remote.rstrip('/') - endpoint = endpoint.lstrip('/') + remote = remote.rstrip("/") + endpoint = endpoint.lstrip("/") - return f'{remote}/{endpoint}' + return f"{remote}/{endpoint}" def _process_qsentry(resp): - if resp.getheader('Content-Encoding', '') == 'gzip': + if resp.getheader("Content-Encoding", "") == "gzip": with GzipFile(fileobj=resp) as file: for line in file.readlines(): yield loads(line) @@ -162,12 +152,12 @@ def search_pmi(search_term, query_type, **kwargs): :rtype: dict """ - kwargs['remote'] = _set_remote('pmi', query_type, **kwargs) - kwargs['token'] = kwargs.get('token', os.getenv('PMI_TOKEN')) + kwargs["remote"] = _set_remote("pmi", query_type, **kwargs) + kwargs["token"] = kwargs.get("token", os.getenv("PMI_TOKEN")) - params = kwargs.get('params', {}) - params.update({'identifier': search_term}) - kwargs['params'] = params + params = kwargs.get("params", {}) + params.update({"identifier": search_term}) + kwargs["params"] = params return loads(_search(**kwargs).read()) @@ -184,13 +174,13 @@ def search_qwatch(search_term, search_type, query_type, **kwargs): :rtype: dict """ - kwargs['remote'] = _set_remote('qwatch', query_type, **kwargs) - kwargs['token'] = kwargs.get('token', os.getenv('QWATCH_TOKEN')) + kwargs["remote"] = _set_remote("qwatch", query_type, **kwargs) + kwargs["token"] = kwargs.get("token", os.getenv("QWATCH_TOKEN")) - params = kwargs.get('params', {}) + params = kwargs.get("params", {}) if search_type: params.update({search_type: search_term}) - kwargs['params'] = params + kwargs["params"] = params return loads(_search(**kwargs).read()) @@ -205,15 +195,15 @@ def search_qauth(search_term, **kwargs): :rtype: dict """ - if not kwargs.get('endpoint'): - kwargs['endpoint'] = '/' + if not kwargs.get("endpoint"): + kwargs["endpoint"] = "/" - kwargs['remote'] = _set_remote('qauth', None, **kwargs) - kwargs['token'] = kwargs.get('token', os.getenv('QAUTH_TOKEN')) + kwargs["remote"] = _set_remote("qauth", None, **kwargs) + kwargs["token"] = kwargs.get("token", os.getenv("QAUTH_TOKEN")) - params = kwargs.get('params', {}) - params.update({'q': search_term}) - kwargs['params'] = params + params = kwargs.get("params", {}) + params.update({"q": search_term}) + kwargs["params"] = params return loads(_search(**kwargs).read()) @@ -228,20 +218,20 @@ def search_qsentry(search_term, **kwargs): :rtype: dict """ - if not kwargs.get('endpoint'): - kwargs['endpoint'] = '/' + if not kwargs.get("endpoint"): + kwargs["endpoint"] = "/" - kwargs['remote'] = _set_remote('qsentry', None, **kwargs) - kwargs['token'] = kwargs.get('token', os.getenv('QSENTRY_TOKEN')) + kwargs["remote"] = _set_remote("qsentry", None, **kwargs) + kwargs["token"] = kwargs.get("token", os.getenv("QSENTRY_TOKEN")) - params = kwargs.get('params', {}) - params.update({'q': search_term}) - kwargs['params'] = params + params = kwargs.get("params", {}) + params.update({"q": search_term}) + kwargs["params"] = params return loads(_search(**kwargs).read()) -def qsentry_feed(query_type='anon', feed_date=datetime.today(), **kwargs): +def qsentry_feed(query_type="anon", feed_date=datetime.today(), **kwargs): """ Fetch the most recent QSentry Feed @@ -252,11 +242,11 @@ def qsentry_feed(query_type='anon', feed_date=datetime.today(), **kwargs): :rtype: Iterator[dict] """ - remote = _set_remote('qsentry_feed', query_type, **kwargs) - kwargs['token'] = kwargs.get('token', os.getenv('QSENTRY_TOKEN')) + remote = _set_remote("qsentry_feed", query_type, **kwargs) + kwargs["token"] = kwargs.get("token", os.getenv("QSENTRY_TOKEN")) - feed_date = (feed_date - timedelta(days=1)).strftime('%Y%m%d') - kwargs['remote'] = f'{remote}/{feed_date}' + feed_date = (feed_date - timedelta(days=1)).strftime("%Y%m%d") + kwargs["remote"] = f"{remote}/{feed_date}" resp = _search(**kwargs) for r in _process_qsentry(resp): diff --git a/misp_modules/lib/vt_graph_parser/__init__.py b/misp_modules/lib/vt_graph_parser/__init__.py index abc02c575..5ca726d1e 100644 --- a/misp_modules/lib/vt_graph_parser/__init__.py +++ b/misp_modules/lib/vt_graph_parser/__init__.py @@ -3,6 +3,5 @@ This module provides methods to import graph from misp. """ - from .helpers import * # noqa from .importers import * # noqa diff --git a/misp_modules/lib/vt_graph_parser/helpers/__init__.py b/misp_modules/lib/vt_graph_parser/helpers/__init__.py index 8f9f66027..d149489d0 100644 --- a/misp_modules/lib/vt_graph_parser/helpers/__init__.py +++ b/misp_modules/lib/vt_graph_parser/helpers/__init__.py @@ -3,5 +3,4 @@ This modules provides functions and attributes to help MISP importers. """ - __all__ = ["parsers", "rules", "wrappers"] diff --git a/misp_modules/lib/vt_graph_parser/helpers/parsers.py b/misp_modules/lib/vt_graph_parser/helpers/parsers.py index 8ca57458a..a06c4ba70 100644 --- a/misp_modules/lib/vt_graph_parser/helpers/parsers.py +++ b/misp_modules/lib/vt_graph_parser/helpers/parsers.py @@ -3,10 +3,8 @@ This module provides parsers for MISP inputs. """ - from vt_graph_parser.helpers.wrappers import MispAttribute - MISP_INPUT_ATTR = [ "hostname", "domain", @@ -19,7 +17,7 @@ "filename|md5", "filename", "target-user", - "target-email" + "target-email", ] VIRUSTOTAL_GRAPH_LINK_PREFIX = "https://www.virustotal.com/graph/" @@ -40,22 +38,20 @@ def _parse_data(attributes, objects): vt_graph_link = "" # Get simple MISP event attributes. - attributes_data += ( - [attr for attr in attributes - if attr.get("type") in MISP_INPUT_ATTR]) + attributes_data += [attr for attr in attributes if attr.get("type") in MISP_INPUT_ATTR] # Get attributes from MISP objects too. if objects: for object_ in objects: object_attrs = object_.get("Attribute", []) - attributes_data += ( - [attr for attr in object_attrs - if attr.get("type") in MISP_INPUT_ATTR]) + attributes_data += [attr for attr in object_attrs if attr.get("type") in MISP_INPUT_ATTR] # Check if there is any VirusTotal Graph computed in MISP event. vt_graph_links = ( - attr for attr in attributes if attr.get("type") == "link" - and attr.get("value", "").startswith(VIRUSTOTAL_GRAPH_LINK_PREFIX)) + attr + for attr in attributes + if attr.get("type") == "link" and attr.get("value", "").startswith(VIRUSTOTAL_GRAPH_LINK_PREFIX) + ) # MISP could have more than one VirusTotal Graph, so we will take # the last one. @@ -66,11 +62,8 @@ def _parse_data(attributes, objects): current_id = int(link.get("id")) vt_graph_link = link.get("value") - attributes = [ - MispAttribute(data["type"], data["category"], data["value"]) - for data in attributes_data] - return (attributes, - vt_graph_link.replace(VIRUSTOTAL_GRAPH_LINK_PREFIX, "")) + attributes = [MispAttribute(data["type"], data["category"], data["value"]) for data in attributes_data] + return (attributes, vt_graph_link.replace(VIRUSTOTAL_GRAPH_LINK_PREFIX, "")) def parse_pymisp_response(payload): diff --git a/misp_modules/lib/vt_graph_parser/helpers/rules.py b/misp_modules/lib/vt_graph_parser/helpers/rules.py index 92ad5d24b..509eb94e7 100644 --- a/misp_modules/lib/vt_graph_parser/helpers/rules.py +++ b/misp_modules/lib/vt_graph_parser/helpers/rules.py @@ -10,7 +10,6 @@ - IP: https://docs.virustotal.com/reference/ip-object#relationships """ - import abc @@ -33,7 +32,7 @@ def __init__(self, last_rule=None, node=None): "ip_address": self.__ip_transition, "url": self.__url_transition, "domain": self.__domain_transition, - "file": self.__file_transition + "file": self.__file_transition, } def get_last_different_rule(self): @@ -153,7 +152,7 @@ def __init__(self, last_rule=None, node=None): "ip_address": self.__ip_transition, "url": self.__url_transition, "domain": self.__domain_transition, - "file": self.__file_transition + "file": self.__file_transition, } def __file_transition(self, graph, node, misp_category): @@ -185,7 +184,7 @@ def __init__(self, last_rule=None, node=None): "ip_address": self.__ip_transition, "url": self.__url_transition, "domain": self.__domain_transition, - "file": self.__file_transition + "file": self.__file_transition, } def __file_transition(self, graph, node, misp_category): @@ -220,7 +219,7 @@ def __init__(self, last_rule=None, node=None): "ip_address": self.__ip_transition, "url": self.__url_transition, "domain": self.__domain_transition, - "file": self.__file_transition + "file": self.__file_transition, } def __file_transition(self, graph, node, misp_category): @@ -256,7 +255,7 @@ def __init__(self, last_rule=None, node=None): "ip_address": self.__ip_transition, "url": self.__url_transition, "domain": self.__domain_transition, - "file": self.__file_transition + "file": self.__file_transition, } def __file_transition(self, graph, node, misp_category): @@ -288,7 +287,7 @@ def __init__(self, last_rule=None, node=None): "ip_address": self.__ip_transition, "url": self.__url_transition, "domain": self.__domain_transition, - "file": self.__file_transition + "file": self.__file_transition, } def __file_transition(self, graph, node, misp_category): diff --git a/misp_modules/lib/vt_graph_parser/helpers/wrappers.py b/misp_modules/lib/vt_graph_parser/helpers/wrappers.py index d376d43b8..fd3944814 100644 --- a/misp_modules/lib/vt_graph_parser/helpers/wrappers.py +++ b/misp_modules/lib/vt_graph_parser/helpers/wrappers.py @@ -27,7 +27,7 @@ class MispAttribute(object): "sha1": "file", "sha256": "file", "target-user": "victim", - "target-email": "email" + "target-email": "email", } def __init__(self, misp_type, category, value, label=""): @@ -52,7 +52,9 @@ def __init__(self, misp_type, category, value, label=""): self.misp_type = misp_type def __eq__(self, other): - return (isinstance(other, self.__class__) and self.value == other.value and self.type == other.type) + return isinstance(other, self.__class__) and self.value == other.value and self.type == other.type def __repr__(self): - return 'MispAttribute("{type}", "{category}", "{value}")'.format(type=self.type, category=self.category, value=self.value) + return 'MispAttribute("{type}", "{category}", "{value}")'.format( + type=self.type, category=self.category, value=self.value + ) diff --git a/misp_modules/lib/vt_graph_parser/importers/__init__.py b/misp_modules/lib/vt_graph_parser/importers/__init__.py index c59197c6b..16e4b73bc 100644 --- a/misp_modules/lib/vt_graph_parser/importers/__init__.py +++ b/misp_modules/lib/vt_graph_parser/importers/__init__.py @@ -3,5 +3,4 @@ This module provides methods to import graphs from MISP. """ - __all__ = ["base", "pymisp_response"] diff --git a/misp_modules/lib/vt_graph_parser/importers/base.py b/misp_modules/lib/vt_graph_parser/importers/base.py index ed5c0fc3f..9f47fd386 100644 --- a/misp_modules/lib/vt_graph_parser/importers/base.py +++ b/misp_modules/lib/vt_graph_parser/importers/base.py @@ -3,16 +3,26 @@ This module provides a common method to import graph from misp attributes. """ - import vt_graph_api from vt_graph_parser.helpers.rules import MispEventInitialRule def import_misp_graph( - misp_attributes, graph_id, vt_api_key, fetch_information, name, - private, fetch_vt_enterprise, user_editors, user_viewers, group_editors, - group_viewers, use_vt_to_connect_the_graph, max_api_quotas, - max_search_depth): + misp_attributes, + graph_id, + vt_api_key, + fetch_information, + name, + private, + fetch_vt_enterprise, + user_editors, + user_viewers, + group_editors, + group_viewers, + use_vt_to_connect_the_graph, + max_api_quotas, + max_search_depth, +): """Import VirusTotal Graph from MISP. Args: @@ -60,38 +70,42 @@ def import_misp_graph( # a new graph will be created. if not graph_id: graph = vt_graph_api.VTGraph( - api_key=vt_api_key, name=name, private=private, - user_editors=user_editors, user_viewers=user_viewers, - group_editors=group_editors, group_viewers=group_viewers) + api_key=vt_api_key, + name=name, + private=private, + user_editors=user_editors, + user_viewers=user_viewers, + group_editors=group_editors, + group_viewers=group_viewers, + ) else: graph = vt_graph_api.VTGraph.load_graph(graph_id, vt_api_key) - attributes_to_add = [attr for attr in misp_attributes - if not graph.has_node(attr.value)] + attributes_to_add = [attr for attr in misp_attributes if not graph.has_node(attr.value)] - total_expandable_attrs = max(sum( - 1 for attr in attributes_to_add - if attr.type in vt_graph_api.Node.SUPPORTED_NODE_TYPES), - 1) + total_expandable_attrs = max( + sum(1 for attr in attributes_to_add if attr.type in vt_graph_api.Node.SUPPORTED_NODE_TYPES), + 1, + ) - max_quotas_per_search = max( - int(max_api_quotas / total_expandable_attrs), 1) + max_quotas_per_search = max(int(max_api_quotas / total_expandable_attrs), 1) previous_node_id = "" for attr in attributes_to_add: # Add the current attr as node to the graph. - added_node = graph.add_node( - attr.value, attr.type, fetch_information, fetch_vt_enterprise, - attr.label) + added_node = graph.add_node(attr.value, attr.type, fetch_information, fetch_vt_enterprise, attr.label) # If use_vt_to_connect_the_grap is True the nodes will be connected using # VT API. if use_vt_to_connect_the_graph: - if (attr.type not in vt_graph_api.Node.SUPPORTED_NODE_TYPES and previous_node_id): + if attr.type not in vt_graph_api.Node.SUPPORTED_NODE_TYPES and previous_node_id: graph.add_link(previous_node_id, attr.value, "manual") else: graph.connect_with_graph( - attr.value, max_quotas_per_search, max_search_depth, - fetch_info_collected_nodes=fetch_information) + attr.value, + max_quotas_per_search, + max_search_depth, + fetch_info_collected_nodes=fetch_information, + ) else: rule = rule.resolve_relation(graph, added_node, attr.category) diff --git a/misp_modules/lib/vt_graph_parser/importers/pymisp_response.py b/misp_modules/lib/vt_graph_parser/importers/pymisp_response.py index e0e834b22..99dc0d408 100644 --- a/misp_modules/lib/vt_graph_parser/importers/pymisp_response.py +++ b/misp_modules/lib/vt_graph_parser/importers/pymisp_response.py @@ -4,17 +4,25 @@ response payload giving by MISP API directly. """ - from vt_graph_parser.helpers.parsers import parse_pymisp_response from vt_graph_parser.importers.base import import_misp_graph def from_pymisp_response( - payload, vt_api_key, fetch_information=True, - private=False, fetch_vt_enterprise=False, user_editors=None, - user_viewers=None, group_editors=None, group_viewers=None, - use_vt_to_connect_the_graph=False, max_api_quotas=1000, - max_search_depth=3, expand_node_one_level=False): + payload, + vt_api_key, + fetch_information=True, + private=False, + fetch_vt_enterprise=False, + user_editors=None, + user_viewers=None, + group_editors=None, + group_viewers=None, + use_vt_to_connect_the_graph=False, + max_api_quotas=1000, + max_search_depth=3, + expand_node_one_level=False, +): """Import VirusTotal Graph from MISP JSON file. Args: @@ -59,14 +67,25 @@ def from_pymisp_response( [vt_graph_api.graph.VTGraph: the imported graph]. """ graphs = [] - for event_payload in payload['data']: + for event_payload in payload["data"]: misp_attrs, graph_id = parse_pymisp_response(event_payload) name = "Graph created from MISP event" graph = import_misp_graph( - misp_attrs, graph_id, vt_api_key, fetch_information, name, - private, fetch_vt_enterprise, user_editors, user_viewers, group_editors, - group_viewers, use_vt_to_connect_the_graph, max_api_quotas, - max_search_depth) + misp_attrs, + graph_id, + vt_api_key, + fetch_information, + name, + private, + fetch_vt_enterprise, + user_editors, + user_viewers, + group_editors, + group_viewers, + use_vt_to_connect_the_graph, + max_api_quotas, + max_search_depth, + ) if expand_node_one_level: graph.expand_n_level(1) graphs.append(graph) diff --git a/misp_modules/modules/__init__.py b/misp_modules/modules/__init__.py index 97fdc13aa..e69de29bb 100644 --- a/misp_modules/modules/__init__.py +++ b/misp_modules/modules/__init__.py @@ -1,4 +0,0 @@ -from .expansion import * # noqa -from .import_mod import * # noqa -from .export_mod import * # noqa -from .action_mod import * # noqa diff --git a/misp_modules/modules/action_mod/__init__.py b/misp_modules/modules/action_mod/__init__.py index b5f6c39fa..e69de29bb 100644 --- a/misp_modules/modules/action_mod/__init__.py +++ b/misp_modules/modules/action_mod/__init__.py @@ -1 +0,0 @@ -__all__ = ['testaction', 'mattermost', 'slack'] diff --git a/misp_modules/modules/action_mod/_utils/__init__.py b/misp_modules/modules/action_mod/_utils/__init__.py index 8b1378917..e69de29bb 100644 --- a/misp_modules/modules/action_mod/_utils/__init__.py +++ b/misp_modules/modules/action_mod/_utils/__init__.py @@ -1 +0,0 @@ - diff --git a/misp_modules/modules/action_mod/_utils/utils.py b/misp_modules/modules/action_mod/_utils/utils.py index 3afdc17bb..a53b471ff 100644 --- a/misp_modules/modules/action_mod/_utils/utils.py +++ b/misp_modules/modules/action_mod/_utils/utils.py @@ -67,4 +67,4 @@ def renderTemplate(data, template=default_template): env = SandboxedEnvironment() - return env.from_string(template).render(data) \ No newline at end of file + return env.from_string(template).render(data) diff --git a/misp_modules/modules/action_mod/mattermost.py b/misp_modules/modules/action_mod/mattermost.py index a25b5f017..3b87a0316 100644 --- a/misp_modules/modules/action_mod/mattermost.py +++ b/misp_modules/modules/action_mod/mattermost.py @@ -1,92 +1,93 @@ import json -from pymisp.tools._psl_faup import PSLFaup as Faup + from mattermostdriver import Driver +from pymisp.tools._psl_faup import PSLFaup as Faup + from ._utils import utils -misperrors = {'error': 'Error'} +misperrors = {"error": "Error"} # config fields that your code expects from the site admin moduleconfig = { - 'params': { - 'mattermost_hostname': { - 'type': 'string', - 'description': 'The Mattermost domain or URL', - 'value': 'example.mattermost.com', + "params": { + "mattermost_hostname": { + "type": "string", + "description": "The Mattermost domain or URL", + "value": "example.mattermost.com", }, - 'bot_access_token': { - 'type': 'string', - 'description': 'Access token generated when you created the bot account', + "bot_access_token": { + "type": "string", + "description": "Access token generated when you created the bot account", }, - 'channel_id': { - 'type': 'string', - 'description': 'The channel you added the bot to', + "channel_id": { + "type": "string", + "description": "The channel you added the bot to", }, - 'message_template': { - 'type': 'large_string', - 'description': 'The template to be used to generate the message to be posted', - 'value': 'The **template** will be rendered using *Jinja2*!', - 'jinja_supported': True, + "message_template": { + "type": "large_string", + "description": "The template to be used to generate the message to be posted", + "value": "The **template** will be rendered using *Jinja2*!", + "jinja_supported": True, }, }, # Blocking modules break the exection of the current of action - 'blocking': False, + "blocking": False, # Indicates whether parts of the data passed to this module should be filtered. Filtered data can be found under the `filteredItems` key - 'support_filters': True, + "support_filters": True, # Indicates whether the data passed to this module should be compliant with the MISP core format - 'expect_misp_core_format': False, + "expect_misp_core_format": False, } # returns either "boolean" or "data" # Boolean is used to simply signal that the execution has finished. # For blocking modules the actual boolean value determines whether we break execution -returns = 'boolean' +returns = "boolean" moduleinfo = { - 'version': '0.1', - 'author': 'Sami Mokaddem', - 'description': 'Simplistic module to send message to a Mattermost channel.', - 'module-type': ['action'], - 'name': 'Mattermost', - 'logo': '', - 'requirements': [], - 'features': '', - 'references': [], - 'input': '', - 'output': '' + "version": "0.1", + "author": "Sami Mokaddem", + "description": "Simplistic module to send message to a Mattermost channel.", + "module-type": ["action"], + "name": "Mattermost", + "logo": "", + "requirements": [], + "features": "", + "references": [], + "input": "", + "output": "", } f = Faup() def createPost(request): - params = request['params'] - f.decode(params['mattermost_hostname']) + params = request["params"] + f.decode(params["mattermost_hostname"]) parsedURL = f.get() - mm = Driver({ - 'url': parsedURL['host'], - 'token': params['bot_access_token'], - 'scheme': parsedURL['scheme'] if parsedURL['scheme'] is not None else 'https', - 'basepath': '/api/v4', - 'port': int(parsedURL['port']) if parsedURL['port'] is not None else 443, - }) + mm = Driver( + { + "url": parsedURL["host"], + "token": params["bot_access_token"], + "scheme": parsedURL["scheme"] if parsedURL["scheme"] is not None else "https", + "basepath": "/api/v4", + "port": int(parsedURL["port"]) if parsedURL["port"] is not None else 443, + } + ) mm.login() data = {} - if 'matchingData' in request: - data = request['matchingData'] + if "matchingData" in request: + data = request["matchingData"] else: - data = request['data'] + data = request["data"] - if params['message_template']: - message = utils.renderTemplate(data, params['message_template']) + if params["message_template"]: + message = utils.renderTemplate(data, params["message_template"]) else: - message = '```\n{}\n```'.format(json.dumps(data)) + message = "```\n{}\n```".format(json.dumps(data)) - mm.posts.create_post(options={ - 'channel_id': params['channel_id'], - 'message': message - }) + mm.posts.create_post(options={"channel_id": params["channel_id"], "message": message}) return True @@ -102,12 +103,12 @@ def handler(q=False): def introspection(): modulesetup = {} try: - modulesetup['config'] = moduleconfig + modulesetup["config"] = moduleconfig except NameError: pass return modulesetup def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/action_mod/slack.py b/misp_modules/modules/action_mod/slack.py index f0bda6b04..caddab160 100644 --- a/misp_modules/modules/action_mod/slack.py +++ b/misp_modules/modules/action_mod/slack.py @@ -1,75 +1,77 @@ import json + from slack_sdk import WebClient from slack_sdk.errors import SlackApiError + from ._utils import utils -misperrors = {'error': 'Error'} +misperrors = {"error": "Error"} # config fields that your code expects from the site admin moduleconfig = { - 'params': { - 'slack_bot_token': { - 'type': 'string', - 'description': 'The Slack bot token generated when you created the bot account', + "params": { + "slack_bot_token": { + "type": "string", + "description": "The Slack bot token generated when you created the bot account", }, - 'channel_id': { - 'type': 'string', - 'description': 'The channel ID you want to post messages to', + "channel_id": { + "type": "string", + "description": "The channel ID you want to post messages to", }, - 'message_template': { - 'type': 'large_string', - 'description': 'The template to be used to generate the message to be posted', - 'value': 'The **template** will be rendered using *Jinja2*!', - 'jinja_supported': True, + "message_template": { + "type": "large_string", + "description": "The template to be used to generate the message to be posted", + "value": "The **template** will be rendered using *Jinja2*!", + "jinja_supported": True, }, }, # Blocking modules break the execution of the current action - 'blocking': False, + "blocking": False, # Indicates whether parts of the data passed to this module should be filtered. - 'support_filters': True, + "support_filters": True, # Indicates whether the data passed to this module should be compliant with the MISP core format - 'expect_misp_core_format': False, + "expect_misp_core_format": False, } # returns either "boolean" or "data" # Boolean is used to simply signal that the execution has finished. # For blocking modules, the actual boolean value determines whether we break execution -returns = 'boolean' +returns = "boolean" moduleinfo = { - 'version': '0.1', - 'author': 'goodlandsecurity', - 'description': 'Simplistic module to send messages to a Slack channel.', - 'module-type': ['action'], - 'name': 'Slack', - 'logo': '', - 'requirements': [], - 'features': '', - 'references': [], - 'input': '', - 'output': '' + "version": "0.1", + "author": "goodlandsecurity", + "description": "Simplistic module to send messages to a Slack channel.", + "module-type": ["action"], + "name": "Slack", + "logo": "", + "requirements": [], + "features": "", + "references": [], + "input": "", + "output": "", } def create_post(request): - params = request['params'] - slack_token = params['slack_bot_token'] - channel_id = params['channel_id'] + params = request["params"] + slack_token = params["slack_bot_token"] + channel_id = params["channel_id"] client = WebClient(token=slack_token) - data = request.get('matchingData', request.get('data', {})) + data = request.get("matchingData", request.get("data", {})) - if params['message_template']: - message = utils.renderTemplate(data, params['message_template']) + if params["message_template"]: + message = utils.renderTemplate(data, params["message_template"]) else: - message = '```\n{}\n```'.format(json.dumps(data)) + message = "```\n{}\n```".format(json.dumps(data)) try: client.chat_postMessage(channel=channel_id, text=message) return True except SlackApiError as e: - error_message = e.response['error'] + error_message = e.response["error"] print(f"Error posting message: {error_message}") return False @@ -85,12 +87,12 @@ def handler(q=False): def introspection(): modulesetup = {} try: - modulesetup['config'] = moduleconfig + modulesetup["config"] = moduleconfig except NameError: pass return modulesetup def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/action_mod/testaction.py b/misp_modules/modules/action_mod/testaction.py index e85a9f174..26a117b13 100644 --- a/misp_modules/modules/action_mod/testaction.py +++ b/misp_modules/modules/action_mod/testaction.py @@ -1,47 +1,43 @@ import json -misperrors = {'error': 'Error'} +misperrors = {"error": "Error"} # config fields that your code expects from the site admin moduleconfig = { - 'params': { - 'foo': { - 'type': 'string', - 'description': 'blablabla', - 'value': 'xyz' - }, - 'Data extraction path': { + "params": { + "foo": {"type": "string", "description": "blablabla", "value": "xyz"}, + "Data extraction path": { # Extracted data can be found under the `matchingData` key - 'type': 'hash_path', - 'description': 'Only post content extracted from this path', - 'value': 'Attribute.{n}.AttributeTag.{n}.Tag.name', + "type": "hash_path", + "description": "Only post content extracted from this path", + "value": "Attribute.{n}.AttributeTag.{n}.Tag.name", }, }, # Blocking modules break the exection of the current of action - 'blocking': False, + "blocking": False, # Indicates whether parts of the data passed to this module should be extracted. Extracted data can be found under the `filteredItems` key - 'support_filters': False, + "support_filters": False, # Indicates whether the data passed to this module should be compliant with the MISP core format - 'expect_misp_core_format': False, + "expect_misp_core_format": False, } # returns either "boolean" or "data" # Boolean is used to simply signal that the execution has finished. # For blocking modules the actual boolean value determines whether we break execution -returns = 'boolean' +returns = "boolean" moduleinfo = { - 'version': '0.1', - 'author': 'Andras Iklody', - 'description': 'This module is merely a test, always returning true. Triggers on event publishing.', - 'module-type': ['action'], - 'name': 'Test action', - 'logo': '', - 'requirements': [], - 'features': '', - 'references': [], - 'input': '', - 'output': '' + "version": "0.1", + "author": "Andras Iklody", + "description": "This module is merely a test, always returning true. Triggers on event publishing.", + "module-type": ["action"], + "name": "Test action", + "logo": "", + "requirements": [], + "features": "", + "references": [], + "input": "", + "output": "", } @@ -57,12 +53,12 @@ def handler(q=False): def introspection(): modulesetup = {} try: - modulesetup['config'] = moduleconfig + modulesetup["config"] = moduleconfig except NameError: pass return modulesetup def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index 529e05e9c..ef6693cb7 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -1,125 +1,4 @@ -import os -import sys - -sys.path.append('{}/lib'.format('/'.join((os.path.realpath(__file__)).split('/')[:-3]))) - -__all__ = [ - 'cuckoo_submit', - 'vmray_submit', - 'circl_passivedns', - 'circl_passivessl', - 'cluster25_expand', - 'countrycode', - 'cve', - 'cve_advanced', - 'cpe', - 'dns', - 'btc_steroids', - 'domaintools', - 'eupi', - 'eql', - 'farsight_passivedns', - 'ipasn', - 'passivetotal', - 'sourcecache', - 'virustotal', - 'shodan', - 'reversedns', - 'geoip_asn', - 'geoip_city', - 'geoip_country', - 'wiki', - 'iprep', - 'threatminer', - 'otx', - 'threatcrowd', - 'vulndb', - 'crowdstrike_falcon', - 'yara_syntax_validator', - 'hashdd', - 'onyphe', - 'onyphe_full', - 'rbl', - 'xforceexchange', - 'sigma_syntax_validator', - 'stix2_pattern_syntax_validator', - 'sigma_queries', - 'dbl_spamhaus', - 'vulners', - 'yara_query', - 'macaddress_io', - 'intel471', - 'backscatter_io', - 'btc_scam_check', - 'hibp', - 'greynoise', - 'macvendors', - 'qrcode', - 'ocr_enrich', - 'pdf_enrich', - 'docx_enrich', - 'xlsx_enrich', - 'pptx_enrich', - 'ods_enrich', - 'odt_enrich', - 'joesandbox_submit', - 'joesandbox_query', - 'urlhaus', - 'virustotal_public', - 'apiosintds', - 'urlscan', - 'securitytrails', - 'apivoid', - 'assemblyline_submit', - 'assemblyline_query', - 'ransomcoindb', - 'malwarebazaar', - 'lastline_query', - 'lastline_submit', - 'sophoslabs_intelix', - 'cytomic_orion', - 'censys_enrich', - 'trustar_enrich', - 'recordedfuture', - 'html_to_markdown', - 'socialscan', - 'passive_ssh', - 'qintel_qsentry', - 'mwdb', - 'hashlookup', - 'mmdb_lookup', - 'ipqs_fraud_and_risk_scoring', - 'clamav', - 'jinja_template_rendering', - 'hyasinsight', - 'variotdbs', - 'crowdsec', - 'extract_url_components', - 'ipinfo', - 'whoisfreaks', - 'ip2locationio', - 'stairwell', - 'google_threat_intelligence', - 'vulnerability_lookup', - 'vysion', - 'mcafee_insights_enrich', - 'threatfox', - 'yeti', - 'abuseipdb', - 'vmware_nsx', - 'sigmf_expand', - 'google_safe_browsing', - 'google_search', - 'whois', - 'triage_submit', - 'virustotal_upload', - 'malshare_upload', - 'convert_markdown_to_pdf', - 'onion_lookup', -] - - -minimum_required_fields = ('type', 'uuid', 'value') +minimum_required_fields = ("type", "uuid", "value") checking_error = 'containing at least a "type" field and a "value" field' standard_error_message = 'This module requires an "attribute" field as input' diff --git a/misp_modules/modules/expansion/_dnsdb_query/dnsdb_query.py b/misp_modules/modules/expansion/_dnsdb_query/dnsdb_query.py index 5df1207a3..edd992e70 100755 --- a/misp_modules/modules/expansion/_dnsdb_query/dnsdb_query.py +++ b/misp_modules/modules/expansion/_dnsdb_query/dnsdb_query.py @@ -20,32 +20,35 @@ import calendar import errno +import json import locale import optparse import os import re import sys import time -import json from io import StringIO try: - from urllib2 import build_opener, Request, ProxyHandler, HTTPError, URLError - from urllib import quote as urllib_quote, urlencode + from urllib import quote as urllib_quote + from urllib import urlencode + + from urllib2 import HTTPError, ProxyHandler, Request, URLError, build_opener except ImportError: - from urllib.request import build_opener, Request, ProxyHandler, HTTPError, URLError - from urllib.parse import quote as urllib_quote, urlencode + from urllib.parse import quote as urllib_quote + from urllib.parse import urlencode + from urllib.request import HTTPError, ProxyHandler, Request, URLError, build_opener -DEFAULT_CONFIG_FILES = filter(os.path.isfile, ('/etc/dnsdb-query.conf', os.path.expanduser('~/.dnsdb-query.conf'))) -DEFAULT_DNSDB_SERVER = 'https://api.dnsdb.info' -DEFAULT_HTTP_PROXY = '' -DEFAULT_HTTPS_PROXY = '' +DEFAULT_CONFIG_FILES = filter(os.path.isfile, ("/etc/dnsdb-query.conf", os.path.expanduser("~/.dnsdb-query.conf"))) +DEFAULT_DNSDB_SERVER = "https://api.dnsdb.info" +DEFAULT_HTTP_PROXY = "" +DEFAULT_HTTPS_PROXY = "" cfg = None options = None -locale.setlocale(locale.LC_ALL, '') +locale.setlocale(locale.LC_ALL, "") class QueryError(Exception): @@ -63,51 +66,51 @@ def __init__(self, server, apikey, limit=None, http_proxy=None, https_proxy=None def query_rrset(self, oname, rrtype=None, bailiwick=None, before=None, after=None): if bailiwick: if not rrtype: - rrtype = 'ANY' - path = 'rrset/name/%s/%s/%s' % (quote(oname), rrtype, quote(bailiwick)) + rrtype = "ANY" + path = "rrset/name/%s/%s/%s" % (quote(oname), rrtype, quote(bailiwick)) elif rrtype: - path = 'rrset/name/%s/%s' % (quote(oname), rrtype) + path = "rrset/name/%s/%s" % (quote(oname), rrtype) else: - path = 'rrset/name/%s' % quote(oname) + path = "rrset/name/%s" % quote(oname) return self._query(path, before, after) def query_rdata_name(self, rdata_name, rrtype=None, before=None, after=None): if rrtype: - path = 'rdata/name/%s/%s' % (quote(rdata_name), rrtype) + path = "rdata/name/%s/%s" % (quote(rdata_name), rrtype) else: - path = 'rdata/name/%s' % quote(rdata_name) + path = "rdata/name/%s" % quote(rdata_name) return self._query(path, before, after) def query_rdata_ip(self, rdata_ip, before=None, after=None): - path = 'rdata/ip/%s' % rdata_ip.replace('/', ',') + path = "rdata/ip/%s" % rdata_ip.replace("/", ",") return self._query(path, before, after) def _query(self, path, before=None, after=None): - url = '%s/lookup/%s' % (self.server, path) + url = "%s/lookup/%s" % (self.server, path) params = {} if self.limit: - params['limit'] = self.limit + params["limit"] = self.limit if before and after: - params['time_first_after'] = after - params['time_last_before'] = before + params["time_first_after"] = after + params["time_last_before"] = before else: if before: - params['time_first_before'] = before + params["time_first_before"] = before if after: - params['time_last_after'] = after + params["time_last_after"] = after if params: - url += '?{0}'.format(urlencode(params)) + url += "?{0}".format(urlencode(params)) req = Request(url) - req.add_header('Accept', 'application/json') - req.add_header('X-Api-Key', self.apikey) + req.add_header("Accept", "application/json") + req.add_header("X-Api-Key", self.apikey) proxy_args = {} if self.http_proxy: - proxy_args['http'] = self.http_proxy + proxy_args["http"] = self.http_proxy if self.https_proxy: - proxy_args['https'] = self.https_proxy + proxy_args["https"] = self.https_proxy proxy_handler = ProxyHandler(proxy_args) opener = build_opener(proxy_handler) @@ -117,7 +120,7 @@ def _query(self, path, before=None, after=None): line = http.readline() if not line: break - yield json.loads(line.decode('ascii')) + yield json.loads(line.decode("ascii")) except (HTTPError, URLError) as e: try: raise QueryError(str(e), sys.exc_traceback) @@ -126,36 +129,36 @@ def _query(self, path, before=None, after=None): def quote(path): - return urllib_quote(path, safe='') + return urllib_quote(path, safe="") def sec_to_text(ts): - return time.strftime('%Y-%m-%d %H:%M:%S -0000', time.gmtime(ts)) + return time.strftime("%Y-%m-%d %H:%M:%S -0000", time.gmtime(ts)) def rrset_to_text(m): s = StringIO() try: - if 'bailiwick' in m: - s.write(';; bailiwick: %s\n' % m['bailiwick']) + if "bailiwick" in m: + s.write(";; bailiwick: %s\n" % m["bailiwick"]) - if 'count' in m: - s.write(';; count: %s\n' % locale.format('%d', m['count'], True)) + if "count" in m: + s.write(";; count: %s\n" % locale.format("%d", m["count"], True)) - if 'time_first' in m: - s.write(';; first seen: %s\n' % sec_to_text(m['time_first'])) - if 'time_last' in m: - s.write(';; last seen: %s\n' % sec_to_text(m['time_last'])) + if "time_first" in m: + s.write(";; first seen: %s\n" % sec_to_text(m["time_first"])) + if "time_last" in m: + s.write(";; last seen: %s\n" % sec_to_text(m["time_last"])) - if 'zone_time_first' in m: - s.write(';; first seen in zone file: %s\n' % sec_to_text(m['zone_time_first'])) - if 'zone_time_last' in m: - s.write(';; last seen in zone file: %s\n' % sec_to_text(m['zone_time_last'])) + if "zone_time_first" in m: + s.write(";; first seen in zone file: %s\n" % sec_to_text(m["zone_time_first"])) + if "zone_time_last" in m: + s.write(";; last seen in zone file: %s\n" % sec_to_text(m["zone_time_last"])) - if 'rdata' in m: - for rdata in m['rdata']: - s.write('%s IN %s %s\n' % (m['rrname'], m['rrtype'], rdata)) + if "rdata" in m: + for rdata in m["rdata"]: + s.write("%s IN %s %s\n" % (m["rrname"], m["rrtype"], rdata)) s.seek(0) return s.read() @@ -164,18 +167,18 @@ def rrset_to_text(m): def rdata_to_text(m): - return '%s IN %s %s' % (m['rrname'], m['rrtype'], m['rdata']) + return "%s IN %s %s" % (m["rrname"], m["rrtype"], m["rdata"]) def parse_config(cfg_files): config = {} if not cfg_files: - raise IOError(errno.ENOENT, 'dnsdb_query: No config files found') + raise IOError(errno.ENOENT, "dnsdb_query: No config files found") for fname in cfg_files: for line in open(fname): - key, eq, val = line.strip().partition('=') + key, eq, val = line.strip().partition("=") val = val.strip('"') config[key] = val @@ -190,24 +193,26 @@ def time_parse(s): pass try: - epoch = int(calendar.timegm(time.strptime(s, '%Y-%m-%d'))) + epoch = int(calendar.timegm(time.strptime(s, "%Y-%m-%d"))) return epoch except ValueError: pass try: - epoch = int(calendar.timegm(time.strptime(s, '%Y-%m-%d %H:%M:%S'))) + epoch = int(calendar.timegm(time.strptime(s, "%Y-%m-%d %H:%M:%S"))) return epoch except ValueError: pass - m = re.match(r'^(?=\d)(?:(\d+)w)?(?:(\d+)d)?(?:(\d+)h)?(?:(\d+)m)?(?:(\d+)s?)?$', s, re.I) + m = re.match(r"^(?=\d)(?:(\d+)w)?(?:(\d+)d)?(?:(\d+)h)?(?:(\d+)m)?(?:(\d+)s?)?$", s, re.I) if m: - return -1 * (int(m.group(1) or 0) * 604800 - + int(m.group(2) or 0) * 86400 - + int(m.group(3) or 0) * 3600 - + int(m.group(4) or 0) * 60 - + int(m.group(5) or 0)) + return -1 * ( + int(m.group(1) or 0) * 604800 + + int(m.group(2) or 0) * 86400 + + int(m.group(3) or 0) * 3600 + + int(m.group(4) or 0) * 60 + + int(m.group(5) or 0) + ) raise ValueError('Invalid time: "%s"' % s) @@ -220,6 +225,7 @@ def f(*args, **kwargs): if e.errno == errno.EPIPE: sys.exit(e.errno) raise + return f @@ -228,20 +234,76 @@ def main(): global cfg global options - parser = optparse.OptionParser(epilog='Time formats are: "%Y-%m-%d", "%Y-%m-%d %H:%M:%S", "%d" (UNIX timestamp), "-%d" (Relative time in seconds), BIND format (e.g. 1w1h, (w)eek, (d)ay, (h)our, (m)inute, (s)econd)') - parser.add_option('-c', '--config', dest='config', help='config file', action='append') - parser.add_option('-r', '--rrset', dest='rrset', type='string', help='rrset [/[/BAILIWICK]]') - parser.add_option('-n', '--rdataname', dest='rdata_name', type='string', help='rdata name [/]') - parser.add_option('-i', '--rdataip', dest='rdata_ip', type='string', help='rdata ip ') - parser.add_option('-t', '--rrtype', dest='rrtype', type='string', help='rrset or rdata rrtype') - parser.add_option('-b', '--bailiwick', dest='bailiwick', type='string', help='rrset bailiwick') - parser.add_option('-s', '--sort', dest='sort', type='string', help='sort key') - parser.add_option('-R', '--reverse', dest='reverse', action='store_true', default=False, help='reverse sort') - parser.add_option('-j', '--json', dest='json', action='store_true', default=False, help='output in JSON format') - parser.add_option('-l', '--limit', dest='limit', type='int', default=0, help='limit number of results') - - parser.add_option('', '--before', dest='before', type='string', help='only output results seen before this time') - parser.add_option('', '--after', dest='after', type='string', help='only output results seen after this time') + parser = optparse.OptionParser( + epilog=( + 'Time formats are: "%Y-%m-%d", "%Y-%m-%d %H:%M:%S", "%d" (UNIX timestamp), "-%d" (Relative time in' + " seconds), BIND format (e.g. 1w1h, (w)eek, (d)ay, (h)our, (m)inute, (s)econd)" + ) + ) + parser.add_option("-c", "--config", dest="config", help="config file", action="append") + parser.add_option( + "-r", + "--rrset", + dest="rrset", + type="string", + help="rrset [/[/BAILIWICK]]", + ) + parser.add_option( + "-n", + "--rdataname", + dest="rdata_name", + type="string", + help="rdata name [/]", + ) + parser.add_option( + "-i", + "--rdataip", + dest="rdata_ip", + type="string", + help="rdata ip ", + ) + parser.add_option("-t", "--rrtype", dest="rrtype", type="string", help="rrset or rdata rrtype") + parser.add_option("-b", "--bailiwick", dest="bailiwick", type="string", help="rrset bailiwick") + parser.add_option("-s", "--sort", dest="sort", type="string", help="sort key") + parser.add_option( + "-R", + "--reverse", + dest="reverse", + action="store_true", + default=False, + help="reverse sort", + ) + parser.add_option( + "-j", + "--json", + dest="json", + action="store_true", + default=False, + help="output in JSON format", + ) + parser.add_option( + "-l", + "--limit", + dest="limit", + type="int", + default=0, + help="limit number of results", + ) + + parser.add_option( + "", + "--before", + dest="before", + type="string", + help="only output results seen before this time", + ) + parser.add_option( + "", + "--after", + dest="after", + type="string", + help="only output results seen after this time", + ) options, args = parser.parse_args() if args: @@ -252,13 +314,13 @@ def main(): if options.before: options.before = time_parse(options.before) except ValueError: - print('Could not parse before: {}'.format(options.before)) + print("Could not parse before: {}".format(options.before)) try: if options.after: options.after = time_parse(options.after) except ValueError: - print('Could not parse after: {}'.format(options.after)) + print("Could not parse after: {}".format(options.after)) try: cfg = parse_config(options.config or DEFAULT_CONFIG_FILES) @@ -266,25 +328,28 @@ def main(): print(str(e), file=sys.stderr) sys.exit(1) - if 'DNSDB_SERVER' not in cfg: - cfg['DNSDB_SERVER'] = DEFAULT_DNSDB_SERVER - if 'HTTP_PROXY' not in cfg: - cfg['HTTP_PROXY'] = DEFAULT_HTTP_PROXY - if 'HTTPS_PROXY' not in cfg: - cfg['HTTPS_PROXY'] = DEFAULT_HTTPS_PROXY - if 'APIKEY' not in cfg: - sys.stderr.write('dnsdb_query: APIKEY not defined in config file\n') + if "DNSDB_SERVER" not in cfg: + cfg["DNSDB_SERVER"] = DEFAULT_DNSDB_SERVER + if "HTTP_PROXY" not in cfg: + cfg["HTTP_PROXY"] = DEFAULT_HTTP_PROXY + if "HTTPS_PROXY" not in cfg: + cfg["HTTPS_PROXY"] = DEFAULT_HTTPS_PROXY + if "APIKEY" not in cfg: + sys.stderr.write("dnsdb_query: APIKEY not defined in config file\n") sys.exit(1) - client = DnsdbClient(cfg['DNSDB_SERVER'], cfg['APIKEY'], - limit=options.limit, - http_proxy=cfg['HTTP_PROXY'], - https_proxy=cfg['HTTPS_PROXY']) + client = DnsdbClient( + cfg["DNSDB_SERVER"], + cfg["APIKEY"], + limit=options.limit, + http_proxy=cfg["HTTP_PROXY"], + https_proxy=cfg["HTTPS_PROXY"], + ) if options.rrset: if options.rrtype or options.bailiwick: qargs = (options.rrset, options.rrtype, options.bailiwick) else: - qargs = (options.rrset.split('/', 2)) + qargs = options.rrset.split("/", 2) results = client.query_rrset(*qargs, before=options.before, after=options.after) fmt_func = rrset_to_text @@ -292,7 +357,7 @@ def main(): if options.rrtype: qargs = (options.rdata_name, options.rrtype) else: - qargs = (options.rdata_name.split('/', 1)) + qargs = options.rdata_name.split("/", 1) results = client.query_rdata_name(*qargs, before=options.before, after=options.after) fmt_func = rdata_to_text @@ -313,15 +378,18 @@ def main(): if options.sort not in results[0]: sort_keys = results[0].keys() sort_keys.sort() - sys.stderr.write('dnsdb_query: invalid sort key "%s". valid sort keys are %s\n' % (options.sort, ', '.join(sort_keys))) + sys.stderr.write( + 'dnsdb_query: invalid sort key "%s". valid sort keys are %s\n' + % (options.sort, ", ".join(sort_keys)) + ) sys.exit(1) results.sort(key=lambda r: r[options.sort], reverse=options.reverse) for res in results: - sys.stdout.write('%s\n' % fmt_func(res)) + sys.stdout.write("%s\n" % fmt_func(res)) except QueryError as e: print(e.message, file=sys.stderr) sys.exit(1) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/misp_modules/modules/expansion/_ransomcoindb/ransomcoindb.py b/misp_modules/modules/expansion/_ransomcoindb/ransomcoindb.py index 26cd2e3ef..2e6d1e670 100755 --- a/misp_modules/modules/expansion/_ransomcoindb/ransomcoindb.py +++ b/misp_modules/modules/expansion/_ransomcoindb/ransomcoindb.py @@ -1,8 +1,10 @@ #!/usr/bin/env python -import requests import logging import os + +import requests + # import pprint copyright = """ @@ -16,17 +18,20 @@ baseurl = "https://ransomcoindb.concinnity-risks.com/api/v1/" user_agent = "ransomcoindb client via python-requests/%s" % requests.__version__ -urls = {'BTC': {'btc': baseurl + 'bin2btc/', - 'md5': baseurl + 'bin2btc/md5/', - 'sha1': baseurl + 'bin2btc/sha1/', - 'sha256': baseurl + 'bin2btc/sha256/', - }, - 'XMR': {'xmr': baseurl + 'bin2crypto/XMR/', - 'md5': baseurl + 'bin2crypto/XMR/md5/', - 'sha1': baseurl + 'bin2crypto/XMR/sha1/', - 'sha256': baseurl + 'bin2crypto/XMR/sha256/', - } - } +urls = { + "BTC": { + "btc": baseurl + "bin2btc/", + "md5": baseurl + "bin2btc/md5/", + "sha1": baseurl + "bin2btc/sha1/", + "sha256": baseurl + "bin2btc/sha256/", + }, + "XMR": { + "xmr": baseurl + "bin2crypto/XMR/", + "md5": baseurl + "bin2crypto/XMR/md5/", + "sha1": baseurl + "bin2crypto/XMR/sha1/", + "sha256": baseurl + "bin2crypto/XMR/sha256/", + }, +} def get_data_by(coin: str, key: str, value: str, api_key: str): @@ -38,59 +43,64 @@ def get_data_by(coin: str, key: str, value: str, api_key: str): # pprint.pprint("api-key: %s" % api_key) - headers = {'x-api-key': api_key, 'content-type': 'application/json'} - headers.update({'User-Agent': user_agent}) + headers = {"x-api-key": api_key, "content-type": "application/json"} + headers.update({"User-Agent": user_agent}) # check first if valid: - valid_coins = ['BTC', 'XMR'] - valid_keys = ['btc', 'md5', 'sha1', 'sha256'] + valid_coins = ["BTC", "XMR"] + valid_keys = ["btc", "md5", "sha1", "sha256"] if coin not in valid_coins or key not in valid_keys: - logging.error("get_data_by_X(): not a valid key parameter. Must be a valid coin (i.e. from %r) and one of: %r" % (valid_coins, valid_keys)) + logging.error( + "get_data_by_X(): not a valid key parameter. Must be a valid coin (i.e. from %r) and one of: %r" + % (valid_coins, valid_keys) + ) return None try: url = urls[coin.upper()][key] logging.debug("url = %s" % url) if not url: - logging.error("Could not find a valid coin/key combination. Must be a valid coin (i.e. from %r) and one of: %r" % (valid_coins, valid_keys)) + logging.error( + "Could not find a valid coin/key combination. Must be a valid coin (i.e. from %r) and one of: %r" + % (valid_coins, valid_keys) + ) return None r = requests.get(url + "%s" % (value), headers=headers) except Exception as ex: logging.error("could not fetch from the service. Error: %s" % str(ex)) if r.status_code != 200: - logging.error("could not fetch from the service. Status code: %s" % - r.status_code) + logging.error("could not fetch from the service. Status code: %s" % r.status_code) return r.json() def get_bin2btc_by_btc(btc_addr: str, api_key: str): - """ Function to fetch the data from the bin2btc/{btc} endpoint """ - return get_data_by('BTC', 'btc', btc_addr, api_key) + """Function to fetch the data from the bin2btc/{btc} endpoint""" + return get_data_by("BTC", "btc", btc_addr, api_key) def get_bin2btc_by_md5(md5: str, api_key: str): - """ Function to fetch the data from the bin2btc/{md5} endpoint """ - return get_data_by('BTC', 'md5', md5, api_key) + """Function to fetch the data from the bin2btc/{md5} endpoint""" + return get_data_by("BTC", "md5", md5, api_key) def get_bin2btc_by_sha1(sha1: str, api_key: str): - """ Function to fetch the data from the bin2btc/{sha1} endpoint """ - return get_data_by('BTC', 'sha1', sha1, api_key) + """Function to fetch the data from the bin2btc/{sha1} endpoint""" + return get_data_by("BTC", "sha1", sha1, api_key) def get_bin2btc_by_sha256(sha256: str, api_key: str): - """ Function to fetch the data from the bin2btc/{sha256} endpoint """ - return get_data_by('BTC', 'sha256', sha256, api_key) + """Function to fetch the data from the bin2btc/{sha256} endpoint""" + return get_data_by("BTC", "sha256", sha256, api_key) if __name__ == "__main__": - """ Just for testing on the cmd line. """ + """Just for testing on the cmd line.""" to_btc = "1KnuC7FdhGuHpvFNxtBpz299Q5QteUdNCq" - api_key = os.getenv('api_key') + api_key = os.getenv("api_key") r = get_bin2btc_by_btc(to_btc, api_key) print(r) r = get_bin2btc_by_md5("abc", api_key) print(r) - r = get_data_by('XMR', 'md5', "452878CD7", api_key) + r = get_data_by("XMR", "md5", "452878CD7", api_key) print(r) diff --git a/misp_modules/modules/expansion/_vulnerability_parser/vulnerability_parser.py b/misp_modules/modules/expansion/_vulnerability_parser/vulnerability_parser.py index 83f515514..715e6f12d 100644 --- a/misp_modules/modules/expansion/_vulnerability_parser/vulnerability_parser.py +++ b/misp_modules/modules/expansion/_vulnerability_parser/vulnerability_parser.py @@ -1,31 +1,30 @@ import json +from typing import Iterator + import requests from pymisp import MISPAttribute, MISPEvent, MISPObject -from typing import Iterator class VulnerabilityMapping: __variot_data_mapping = { - 'credits': 'credit', - 'description': 'description', - 'title': 'summary' - } - __variot_flat_mapping = { - 'cve': 'id', 'id': 'id' + "credits": "credit", + "description": "description", + "title": "summary", } + __variot_flat_mapping = {"cve": "id", "id": "id"} @classmethod def exploit_mapping(cls) -> dict: return cls.__exploit_mapping - + @classmethod def exploit_multiple_mapping(cls) -> dict: return cls.__exploit_multiple_mapping - + @classmethod def variot_data_mapping(cls) -> dict: return cls.__variot_data_mapping - + @classmethod def variot_flat_mapping(cls) -> dict: return cls.__variot_flat_mapping @@ -43,162 +42,137 @@ def __init__(self, attribute: dict): @property def misp_attribute(self): return self.__misp_attribute - + @property def misp_event(self): return self.__misp_event def get_results(self) -> dict: event = json.loads(self.misp_event.to_json()) - return { - 'results': { - key: value for key, value in event.items() - if key in ('Attribute', 'Object') - } - } + return {"results": {key: value for key, value in event.items() if key in ("Attribute", "Object")}} def _parse_variot_description(self, query_results): - vulnerability_object = MISPObject('vulnerability') + vulnerability_object = MISPObject("vulnerability") for field, relation in self.mapping.variot_flat_mapping().items(): if query_results.get(field): - vulnerability_object.add_attribute( - relation, query_results[field] - ) + vulnerability_object.add_attribute(relation, query_results[field]) for field, relation in self.mapping.variot_data_mapping().items(): - if query_results.get(field, {}).get('data'): - vulnerability_object.add_attribute( - relation, query_results[field]['data'] - ) - if query_results.get('configurations', {}).get('data'): - for configuration in query_results['configurations']['data']: - for node in configuration['nodes']: - for cpe_match in node['cpe_match']: - if cpe_match['vulnerable']: - vulnerability_object.add_attribute( - 'vulnerable-configuration', - cpe_match['cpe23Uri'] - ) - if query_results.get('cvss', {}).get('data'): + if query_results.get(field, {}).get("data"): + vulnerability_object.add_attribute(relation, query_results[field]["data"]) + if query_results.get("configurations", {}).get("data"): + for configuration in query_results["configurations"]["data"]: + for node in configuration["nodes"]: + for cpe_match in node["cpe_match"]: + if cpe_match["vulnerable"]: + vulnerability_object.add_attribute("vulnerable-configuration", cpe_match["cpe23Uri"]) + if query_results.get("cvss", {}).get("data"): cvss = {} - for cvss_data in query_results['cvss']['data']: - for cvss_v2 in cvss_data['cvssV2']: - cvss[float(cvss_v2['trust'])] = cvss_v2 - for cvss_v3 in cvss_data['cvssV3']: - cvss[float(cvss_v3['trust'])] = cvss_v3 + for cvss_data in query_results["cvss"]["data"]: + for cvss_v2 in cvss_data["cvssV2"]: + cvss[float(cvss_v2["trust"])] = cvss_v2 + for cvss_v3 in cvss_data["cvssV3"]: + cvss[float(cvss_v3["trust"])] = cvss_v3 if cvss: cvss = cvss[max(cvss)] - vulnerability_object.add_attribute( - 'cvss-score', cvss['baseScore'] - ) - vulnerability_object.add_attribute( - 'cvss-string', cvss['vectorString'] - ) - if query_results.get('references', {}).get('data'): - for reference in query_results['references']['data']: - vulnerability_object.add_attribute( - 'references', reference['url'] - ) - if query_results.get('sources_release_date', {}).get('data'): - for release_date in query_results['sources_release_date']['data']: - if release_date['db'] != 'NVD': + vulnerability_object.add_attribute("cvss-score", cvss["baseScore"]) + vulnerability_object.add_attribute("cvss-string", cvss["vectorString"]) + if query_results.get("references", {}).get("data"): + for reference in query_results["references"]["data"]: + vulnerability_object.add_attribute("references", reference["url"]) + if query_results.get("sources_release_date", {}).get("data"): + for release_date in query_results["sources_release_date"]["data"]: + if release_date["db"] != "NVD": continue - if release_date['id'] == self.misp_attribute.value: - vulnerability_object.add_attribute( - 'published', release_date['date'] - ) + if release_date["id"] == self.misp_attribute.value: + vulnerability_object.add_attribute("published", release_date["date"]) break - if query_results.get('sources_update_date', {}).get('data'): - for update_date in query_results['sources_update_date']['data']: - if update_date['db'] != 'NVD': + if query_results.get("sources_update_date", {}).get("data"): + for update_date in query_results["sources_update_date"]["data"]: + if update_date["db"] != "NVD": continue - if update_date['id'] == self.misp_attribute.value: - vulnerability_object.add_attribute( - 'modified', update_date['date'] - ) + if update_date["id"] == self.misp_attribute.value: + vulnerability_object.add_attribute("modified", update_date["date"]) break - vulnerability_object.add_reference( - self.misp_attribute.uuid, 'related-to' - ) + vulnerability_object.add_reference(self.misp_attribute.uuid, "related-to") self.misp_event.add_object(vulnerability_object) class VulnerabilityLookupMapping(VulnerabilityMapping): __csaf_mapping = { - 'id': 'id', - 'initial_release_date': 'published', - 'current_release_date': 'modified' + "id": "id", + "initial_release_date": "published", + "current_release_date": "modified", } __cve_mapping = { - 'cveId': 'id', - 'datePublished': 'published', - 'dateUpdated': 'modified', - 'state': 'state' - } - __cwe_mapping = { - 'cweId': 'id', - 'description': 'description', - 'name': 'name' - } - __gsd_mapping = { - 'id': 'id', - 'details': 'description', - 'modified': 'modified' + "cveId": "id", + "datePublished": "published", + "dateUpdated": "modified", + "state": "state", } + __cwe_mapping = {"cweId": "id", "description": "description", "name": "name"} + __gsd_mapping = {"id": "id", "details": "description", "modified": "modified"} __jvn_mapping = { - 'sec:identifier': 'id', - 'description': 'description', - 'title': 'summary', - 'link': 'references', - 'dcterms:issued': 'published', - 'dcterms:modified': 'modified' - } - __nvd_mapping = { - 'id': 'id', - 'published': 'published', - 'lastModified': 'modified' + "sec:identifier": "id", + "description": "description", + "title": "summary", + "link": "references", + "dcterms:issued": "published", + "dcterms:modified": "modified", } + __nvd_mapping = {"id": "id", "published": "published", "lastModified": "modified"} __ossf_mapping = { - 'id': 'id', - 'summary': 'summary', - 'details': 'description', - 'published': 'published', - 'modified': 'modified' + "id": "id", + "summary": "summary", + "details": "description", + "published": "published", + "modified": "modified", } __related_vuln_mapping = { - 'cve': 'id', - 'title': 'summary', - 'discovery_date': 'published' + "cve": "id", + "title": "summary", + "discovery_date": "published", } __source_mapping = { - 'cve': '_parse_cve_description', - 'ghsa': '_parse_standard_description', - 'gsd': '_parse_gsd_description', - 'jvndb': '_parse_jvn_description', - 'mal': '_parse_ossf_description', - 'pysec': '_parse_standard_description', - 'ts': '_parse_tailscale_description', - 'var': '_parse_variot_description' + "cve": "_parse_cve_description", + "ghsa": "_parse_standard_description", + "gsd": "_parse_gsd_description", + "jvndb": "_parse_jvn_description", + "mal": "_parse_ossf_description", + "pysec": "_parse_standard_description", + "ts": "_parse_tailscale_description", + "var": "_parse_variot_description", } __source_mapping.update( dict.fromkeys( ( - 'cisco', 'icsa', 'icsma', 'ncsc', 'nn', 'oxas', - 'rhba', 'rhea', 'rhsa', 'sca', 'ssa', 'va', 'wid' + "cisco", + "icsa", + "icsma", + "ncsc", + "nn", + "oxas", + "rhba", + "rhea", + "rhsa", + "sca", + "ssa", + "va", + "wid", ), - '_parse_csaf_description' + "_parse_csaf_description", ) ) __standard_mapping = { - 'id': 'id', - 'details': 'description', - 'published': 'published', - 'modified': 'modified' + "id": "id", + "details": "description", + "published": "published", + "modified": "modified", } __tailscale_mapping = { - 'title': 'id', - 'link': 'references', - 'summary': 'summary', - 'published': 'published' + "title": "id", + "link": "references", + "summary": "summary", + "published": "published", } @classmethod @@ -266,139 +240,122 @@ def mapping(self) -> VulnerabilityLookupMapping: return self.__mapping def parse_lookup_result(self, lookup_result: dict): - feature = self.mapping.source_mapping( - self.misp_attribute.value.split('-')[0].lower() - ) + feature = self.mapping.source_mapping(self.misp_attribute.value.split("-")[0].lower()) getattr(self, feature)(lookup_result) def _create_vulnerability_object(self) -> MISPObject: - misp_object = MISPObject('vulnerability') - misp_object.add_attribute( - 'references', f'{self.api_url}/vuln/{self.misp_attribute.value}' - ) + misp_object = MISPObject("vulnerability") + misp_object.add_attribute("references", f"{self.api_url}/vuln/{self.misp_attribute.value}") return misp_object def _parse_aliases(self, *aliases: tuple) -> Iterator[str]: for alias in aliases: query = requests.get(f"{self.api_url}/api/vulnerability/{alias}") if query.status_code != 200: - self.errors.append( - f'Unable to query related vulnerability id {alias}' - ) + self.errors.append(f"Unable to query related vulnerability id {alias}") continue vulnerability = query.json() if not vulnerability: - self.errors.append( - f'No results for related vulnerability id{alias}' - ) + self.errors.append(f"No results for related vulnerability id{alias}") continue - feature = self.mapping.source_mapping(alias.split('-')[0].lower()) + feature = self.mapping.source_mapping(alias.split("-")[0].lower()) yield getattr(self, feature)(vulnerability) def _parse_csaf_branch(self, branch: list) -> Iterator[str]: for sub_branch in branch: - if sub_branch.get('branches'): - yield from self._parse_csaf_branch(sub_branch['branches']) + if sub_branch.get("branches"): + yield from self._parse_csaf_branch(sub_branch["branches"]) else: - cpe = sub_branch.get('product', {}).get('product_identification_helper', {}).get('cpe') + cpe = sub_branch.get("product", {}).get("product_identification_helper", {}).get("cpe") if cpe is not None: yield cpe def _parse_csaf_description(self, lookup_result: dict) -> str: - description = lookup_result['document'] + description = lookup_result["document"] - tracking = description['tracking'] + tracking = description["tracking"] misp_object = self._create_vulnerability_object() for field, relation in self.mapping.csaf_mapping().items(): misp_object.add_attribute(relation, tracking[field]) - misp_object.add_attribute('summary', description['title']) - for reference in description.get('references', []): - misp_object.add_attribute('references', reference['url']) - misp_object.add_attribute('credit', description['publisher']['name']) - branches = lookup_result.get('product_tree', {}).get('branches', []) + misp_object.add_attribute("summary", description["title"]) + for reference in description.get("references", []): + misp_object.add_attribute("references", reference["url"]) + misp_object.add_attribute("credit", description["publisher"]["name"]) + branches = lookup_result.get("product_tree", {}).get("branches", []) if branches: for cpe in set(self._parse_csaf_branch(branches)): - misp_object.add_attribute('vulnerable-configuration', cpe) - misp_object.add_reference(self.misp_attribute.uuid, 'describes') + misp_object.add_attribute("vulnerable-configuration", cpe) + misp_object.add_reference(self.misp_attribute.uuid, "describes") vulnerability_object = self.misp_event.add_object(misp_object) - for vulnerability in lookup_result['vulnerabilities']: - related = MISPObject('vulnerability') + for vulnerability in lookup_result["vulnerabilities"]: + related = MISPObject("vulnerability") for field, relation in self.mapping.related_vuln_mapping().items(): if vulnerability.get(field): related.add_attribute(relation, vulnerability[field]) - for score in vulnerability.get('scores', []): - cvss_v3 = score['cvss_v3'] - related.add_attribute('cvss-score', cvss_v3['baseScore']) - related.add_attribute('cvss-string', cvss_v3['vectorString']) - for reference in vulnerability.get('references', []): - related.add_attribute('references', reference['url']) - related.add_reference(vulnerability_object.uuid, 'related-to') + for score in vulnerability.get("scores", []): + cvss_v3 = score["cvss_v3"] + related.add_attribute("cvss-score", cvss_v3["baseScore"]) + related.add_attribute("cvss-string", cvss_v3["vectorString"]) + for reference in vulnerability.get("references", []): + related.add_attribute("references", reference["url"]) + related.add_reference(vulnerability_object.uuid, "related-to") related_vulnerability = self.misp_event.add_object(related) - if vulnerability.get('cwe'): - cwe = vulnerability['cwe'] - weakness = MISPObject('weakness') + if vulnerability.get("cwe"): + cwe = vulnerability["cwe"] + weakness = MISPObject("weakness") for field, value in cwe.items(): weakness.add_attribute(field, value) self.misp_event.add_object(weakness) - related_vulnerability.add_reference( - weakness.uuid, 'weakened-by' - ) + related_vulnerability.add_reference(weakness.uuid, "weakened-by") return vulnerability_object.uuid def _parse_cve_description(self, lookup_result: dict) -> str: misp_object = self._create_vulnerability_object() - cveMetaData = lookup_result['cveMetadata'] + cveMetaData = lookup_result["cveMetadata"] for field, relation in self.mapping.cve_mapping().items(): misp_object.add_attribute(relation, cveMetaData[field]) - containers = lookup_result['containers'] - for reference in containers.get('cna', {}).get('references', []): - misp_object.add_attribute('references', reference['url']) - for adp in containers.get('adp', []): - for affected in adp.get('affected', []): - for cpe in affected.get('cpes', []): - misp_object.add_attribute('vulnerable-configuration', cpe) - misp_object.add_reference(self.misp_attribute.uuid, 'related-to') + containers = lookup_result["containers"] + for reference in containers.get("cna", {}).get("references", []): + misp_object.add_attribute("references", reference["url"]) + for adp in containers.get("adp", []): + for affected in adp.get("affected", []): + for cpe in affected.get("cpes", []): + misp_object.add_attribute("vulnerable-configuration", cpe) + misp_object.add_reference(self.misp_attribute.uuid, "related-to") vulnerability_object = self.misp_event.add_object(misp_object) return vulnerability_object.uuid def _parse_cve_related_description(self, cve_description: dict) -> str: - misp_object = MISPObject('vulnerability') - misp_object.add_attribute( - 'id', cve_description['CVE_data_meta']['ID'] - ) + misp_object = MISPObject("vulnerability") + misp_object.add_attribute("id", cve_description["CVE_data_meta"]["ID"]) misp_object.add_attribute( - 'description', - cve_description['description']['description_data'][0]['value'] + "description", + cve_description["description"]["description_data"][0]["value"], ) - for cvss in cve_description.get('impact', {}).get('cvss', []): - misp_object.add_attribute('cvss-score', cvss['baseScore']) - misp_object.add_attribute('cvss-string', cvss['vectorString']) - for reference in misp_object.get('references', {}).get('reference_data', []): - misp_object.add_attribute('references', reference['url']) + for cvss in cve_description.get("impact", {}).get("cvss", []): + misp_object.add_attribute("cvss-score", cvss["baseScore"]) + misp_object.add_attribute("cvss-string", cvss["vectorString"]) + for reference in misp_object.get("references", {}).get("reference_data", []): + misp_object.add_attribute("references", reference["url"]) return self.misp_event.add_object(misp_object).uuid def _parse_gsd_description(self, lookup_result: dict) -> str: misp_object = self._create_vulnerability_object() - gsd_details = lookup_result['gsd']['osvSchema'] + gsd_details = lookup_result["gsd"]["osvSchema"] for field, relation in self.mapping.gsd_mapping().items(): if gsd_details.get(field): misp_object.add_attribute(relation, gsd_details[field]) - misp_object.add_reference(self.misp_attribute.uuid, 'related-to') + misp_object.add_reference(self.misp_attribute.uuid, "related-to") vulnerability_object = self.misp_event.add_object(misp_object) - for field, values in lookup_result['namespaces'].items(): - if field == 'cve.org': - vulnerability_object.add_reference( - self._parse_cve_related_description(values), 'related-to' - ) + for field, values in lookup_result["namespaces"].items(): + if field == "cve.org": + vulnerability_object.add_reference(self._parse_cve_related_description(values), "related-to") continue - if field == 'nvd.nist.gov' and values.get('cve'): - vulnerability_object.add_reference( - self._parse_nvd_related_description(values['cve']), - 'related-to' - ) + if field == "nvd.nist.gov" and values.get("cve"): + vulnerability_object.add_reference(self._parse_nvd_related_description(values["cve"]), "related-to") return vulnerability_object.uuid @@ -406,78 +363,67 @@ def _parse_jvn_description(self, lookup_result: dict) -> str: vulnerability = self._create_vulnerability_object() for field, relation in self.mapping.jvn_mapping().items(): vulnerability.add_attribute(relation, lookup_result[field]) - for cpe in lookup_result.get('sec:cpe', []): - cpe_value = cpe.get('#text') + for cpe in lookup_result.get("sec:cpe", []): + cpe_value = cpe.get("#text") if cpe_value is not None: - vulnerability.add_attribute('vulnerable-configuration', cpe_value) + vulnerability.add_attribute("vulnerable-configuration", cpe_value) misp_object = self.misp_event.add_object(vulnerability) - for reference in lookup_result.get('sec:references', []): - source = reference.get('@source') - if source is None and reference.get('@id', '').startswith('CWE-'): - title = reference.get('@title') + for reference in lookup_result.get("sec:references", []): + source = reference.get("@source") + if source is None and reference.get("@id", "").startswith("CWE-"): + title = reference.get("@title") if title is not None: - weakness = MISPObject('weakness') - weakness.add_attribute('id', reference['@id']) - weakness.add_attribute('description', title) - misp_object.add_reference( - self.misp_event.add_object(weakness).uuid, 'weakened-by' - ) + weakness = MISPObject("weakness") + weakness.add_attribute("id", reference["@id"]) + weakness.add_attribute("description", title) + misp_object.add_reference(self.misp_event.add_object(weakness).uuid, "weakened-by") else: misp_object.add_reference( - self.misp_event.add_attribute( - type='weakness', value=reference['@id']).uuid, - 'weakened-by' + self.misp_event.add_attribute(type="weakness", value=reference["@id"]).uuid, + "weakened-by", ) continue - if source == 'JVN': - misp_object.add_attribute('references', reference['#text']) - elif source == 'CVE': - for referenced_uuid in self._parse_aliases(reference['@id']): - misp_object.add_reference(referenced_uuid, 'related-to') + if source == "JVN": + misp_object.add_attribute("references", reference["#text"]) + elif source == "CVE": + for referenced_uuid in self._parse_aliases(reference["@id"]): + misp_object.add_reference(referenced_uuid, "related-to") return misp_object.uuid def _parse_nvd_related_description(self, nvd_description: dict) -> str: - misp_object = MISPObject('vulnerability') + misp_object = MISPObject("vulnerability") for field, relation in self.mapping.nvd_mapping().items(): misp_object.add_attribute(relation, nvd_description[field]) - misp_object.add_attribute( - 'description', nvd_description['descriptions'][0]['value'] - ) - for cvss in nvd_description.get('metrics', {}).get('cvssMetricV31', []): - misp_object.add_attribute( - 'cvss-score', cvss['cvssData']['baseScore'] - ) - misp_object.add_attribute( - 'cvss-string', cvss['cvssData']['vectorString'] - ) - for reference in nvd_description.get('references', []): - misp_object.add_attribute('references', reference['url']) + misp_object.add_attribute("description", nvd_description["descriptions"][0]["value"]) + for cvss in nvd_description.get("metrics", {}).get("cvssMetricV31", []): + misp_object.add_attribute("cvss-score", cvss["cvssData"]["baseScore"]) + misp_object.add_attribute("cvss-string", cvss["cvssData"]["vectorString"]) + for reference in nvd_description.get("references", []): + misp_object.add_attribute("references", reference["url"]) return self.misp_event.add_object(misp_object).uuid def _parse_ossf_description(self, lookup_result: dict) -> str: misp_object = self._create_vulnerability_object() for field, relation in self.mapping.ossf_mapping().items(): misp_object.add_attribute(relation, lookup_result[field]) - for reference in lookup_result['references']: - misp_object.add_attribute('references', reference['url']) - misp_object.add_reference(self.misp_attribute.uuid, 'related-to') + for reference in lookup_result["references"]: + misp_object.add_attribute("references", reference["url"]) + misp_object.add_reference(self.misp_attribute.uuid, "related-to") vulnerability_object = self.misp_event.add_object(misp_object) - for affected in lookup_result.get('affected', []): - for cwe in affected.get('database_specific', {}).get('cwes', []): - cwe_id = cwe.get('cweId') + for affected in lookup_result.get("affected", []): + for cwe in affected.get("database_specific", {}).get("cwes", []): + cwe_id = cwe.get("cweId") if cwe_id is not None: - weakness = MISPObject('weakness') + weakness = MISPObject("weakness") for field, relation in self.mapping.cwe_mapping().items(): if cwe.get(field): weakness.add_attribute(relation, cwe[field]) self.misp_event.add_object(weakness) - vulnerability_object.add_reference( - weakness.uuid, 'weakened-by' - ) + vulnerability_object.add_reference(weakness.uuid, "weakened-by") - if lookup_result.get('aliases'): - for vuln_uuid in self._parse_aliases(*lookup_result['aliases']): - vulnerability_object.add_reference(vuln_uuid, 'related-to') + if lookup_result.get("aliases"): + for vuln_uuid in self._parse_aliases(*lookup_result["aliases"]): + vulnerability_object.add_reference(vuln_uuid, "related-to") return vulnerability_object.uuid @@ -485,21 +431,19 @@ def _parse_standard_description(self, lookup_result: dict) -> str: misp_object = self._create_vulnerability_object() for field, relation in self.mapping.standard_mapping().items(): misp_object.add_attribute(relation, lookup_result[field]) - for cvss in lookup_result.get('severity', []): - misp_object.add_attribute('cvss-string', cvss['score']) - for reference in lookup_result['references']: - misp_object.add_attribute('references', reference['url']) - for cwe_id in lookup_result.get('database_specific', {}).get('cwe_ids', []): - attribute = self.misp_event.add_attribute( - type='weakness', value=cwe_id - ) - misp_object.add_reference(attribute.uuid, 'weakened-by') - misp_object.add_reference(self.misp_attribute.uuid, 'related-to') + for cvss in lookup_result.get("severity", []): + misp_object.add_attribute("cvss-string", cvss["score"]) + for reference in lookup_result["references"]: + misp_object.add_attribute("references", reference["url"]) + for cwe_id in lookup_result.get("database_specific", {}).get("cwe_ids", []): + attribute = self.misp_event.add_attribute(type="weakness", value=cwe_id) + misp_object.add_reference(attribute.uuid, "weakened-by") + misp_object.add_reference(self.misp_attribute.uuid, "related-to") vulnerability_object = self.misp_event.add_object(misp_object) - if lookup_result.get('aliases'): - for vuln_uuid in self._parse_aliases(*lookup_result['aliases']): - vulnerability_object.add_reference(vuln_uuid, 'related-to') + if lookup_result.get("aliases"): + for vuln_uuid in self._parse_aliases(*lookup_result["aliases"]): + vulnerability_object.add_reference(vuln_uuid, "related-to") return vulnerability_object.uuid @@ -507,5 +451,5 @@ def _parse_tailscale_description(self, lookup_result: dict) -> str: misp_object = self._create_vulnerability_object() for field, relation in self.mapping.tailscale_mapping().items(): misp_object.add_attribute(relation, lookup_result[field]) - misp_object.add_reference(self.misp_attribute.uuid, 'related-to') + misp_object.add_reference(self.misp_attribute.uuid, "related-to") self.misp_event.add_object(misp_object) diff --git a/misp_modules/modules/expansion/abuseipdb.py b/misp_modules/modules/expansion/abuseipdb.py index ba09a4c17..00ef0aa28 100644 --- a/misp_modules/modules/expansion/abuseipdb.py +++ b/misp_modules/modules/expansion/abuseipdb.py @@ -1,26 +1,32 @@ -import requests import json -from pymisp import MISPObject, MISPAttribute, MISPEvent -from . import check_input_attribute, checking_error, standard_error_message + import dns.resolver +import requests +from pymisp import MISPEvent, MISPObject + +from . import check_input_attribute, checking_error, standard_error_message -misperrors = {'error': 'Error'} -mispattributes = {'input': ['ip-src', 'ip-dst', 'hostname', 'domain', 'domain|ip'], 'format': 'misp_standard'} +misperrors = {"error": "Error"} +mispattributes = { + "input": ["ip-src", "ip-dst", "hostname", "domain", "domain|ip"], + "format": "misp_standard", +} moduleinfo = { - 'version': '0.1', - 'author': 'Stephanie S', - 'description': 'AbuseIPDB MISP expansion module', - 'module-type': ['expansion', 'hover'], - 'name': 'Abuse IPDB', - 'logo': '', - 'requirements': [], - 'features': '', - 'references': [], - 'input': '', - 'output': '', + "version": "0.1", + "author": "Stephanie S", + "description": "AbuseIPDB MISP expansion module", + "module-type": ["expansion", "hover"], + "name": "Abuse IPDB", + "logo": "", + "requirements": [], + "features": "", + "references": [], + "input": "", + "output": "", } -moduleconfig = ['api_key', 'max_age_in_days', 'abuse_threshold'] +moduleconfig = ["api_key", "max_age_in_days", "abuse_threshold"] + def get_ip(request): # Need to get the ip from the domain @@ -29,18 +35,19 @@ def get_ip(request): resolver.lifetime = 2 try: - ip = resolver.query(request["attribute"]["value"], 'A') + ip = resolver.query(request["attribute"]["value"], "A") return ip except dns.resolver.NXDOMAIN: - misperrors['error'] = "NXDOMAIN" + misperrors["error"] = "NXDOMAIN" return misperrors except dns.exception.Timeout: - misperrors['error'] = "Timeout" + misperrors["error"] = "Timeout" return misperrors except Exception: - misperrors['error'] = "DNS resolving error" + misperrors["error"] = "DNS resolving error" return misperrors + def handler(q=False): if q is False: return False @@ -52,95 +59,105 @@ def handler(q=False): return {"error": "AbuseIPDB max age in days is missing"} if "abuse_threshold" not in request["config"]: return {"error": "AbuseIPDB abuse threshold is missing"} - if not request.get('attribute') or not check_input_attribute(request['attribute'], requirements=('type', 'value')): - return {'error': f'{standard_error_message}, {checking_error}.'} - if request['attribute']['type'] not in mispattributes['input']: - return {'error': 'Unsupported attribute type.'} - - if (request['attribute']['type'] == 'hostname' or request['attribute']['type'] == 'domain' or request['attribute']['type'] == 'domain|ip'): - ip = get_ip(request)[0] + if not request.get("attribute") or not check_input_attribute(request["attribute"], requirements=("type", "value")): + return {"error": f"{standard_error_message}, {checking_error}."} + if request["attribute"]["type"] not in mispattributes["input"]: + return {"error": "Unsupported attribute type."} + + if ( + request["attribute"]["type"] == "hostname" + or request["attribute"]["type"] == "domain" + or request["attribute"]["type"] == "domain|ip" + ): + ip = get_ip(request)[0] else: ip = request["attribute"]["value"] api_key = request["config"]["api_key"] max_age_in_days = request["config"]["max_age_in_days"] - api_endpoint = 'https://api.abuseipdb.com/api/v2/check' - querystring = { - 'ipAddress': ip, - 'maxAgeInDays': max_age_in_days - } - headers = { - 'Accept': 'application/json', - 'key': api_key - } + api_endpoint = "https://api.abuseipdb.com/api/v2/check" + querystring = {"ipAddress": ip, "maxAgeInDays": max_age_in_days} + headers = {"Accept": "application/json", "key": api_key} r = {"results": []} - response = requests.request(method='GET', url=api_endpoint, headers=headers, params=querystring) + response = requests.request(method="GET", url=api_endpoint, headers=headers, params=querystring) - if (response.status_code == 200): + if response.status_code == 200: response_json = json.loads(response.text) - is_whitelisted = response_json['data']['isWhitelisted'] - is_tor = response_json['data']['isTor'] - is_public = response_json['data']['isPublic'] - abuse_confidence_score = response_json['data']['abuseConfidenceScore'] + is_whitelisted = response_json["data"]["isWhitelisted"] + is_tor = response_json["data"]["isTor"] + is_public = response_json["data"]["isPublic"] + abuse_confidence_score = response_json["data"]["abuseConfidenceScore"] abuse_threshold = request["config"]["abuse_threshold"] - if (request["config"]["abuse_threshold"] is not None): + if request["config"]["abuse_threshold"] is not None: abuse_threshold = request["config"]["abuse_threshold"] else: abuse_threshold = 70 - if (is_whitelisted == False): + if is_whitelisted == False: is_whitelisted = 0 - if (is_tor == False): + if is_tor == False: is_tor = 0 - if (is_public == False): + if is_public == False: is_public = 0 - if (abuse_confidence_score == None): + if abuse_confidence_score is None: abuse_confidence_score = 0 - if (response_json.get("errors")): - return {'error': 'AbuseIPDB error, check logs'} + if response_json.get("errors"): + return {"error": "AbuseIPDB error, check logs"} else: event = MISPEvent() - obj = MISPObject('abuseipdb') - event.add_attribute(**request['attribute']) + obj = MISPObject("abuseipdb") + event.add_attribute(**request["attribute"]) if int(abuse_confidence_score) >= int(abuse_threshold): - malicious_attribute = obj.add_attribute('is-malicious', **{'type': 'boolean', 'value': 1}) - malicious_attribute.add_tag(f'ioc:artifact-state="malicious"') + malicious_attribute = obj.add_attribute("is-malicious", **{"type": "boolean", "value": 1}) + malicious_attribute.add_tag('ioc:artifact-state="malicious"') else: - malicious_attribute = obj.add_attribute('is-malicious', **{'type': 'boolean', 'value': 0}) - malicious_attribute.add_tag(f'ioc:artifact-state="not-malicious"') + malicious_attribute = obj.add_attribute("is-malicious", **{"type": "boolean", "value": 0}) + malicious_attribute.add_tag('ioc:artifact-state="not-malicious"') if is_whitelisted is not None: - obj.add_attribute('is-whitelisted', **{'type': 'boolean', 'value': is_whitelisted}) - obj.add_attribute('is-tor', **{'type': 'boolean', 'value': is_tor}) - obj.add_attribute('is-public', **{'type': 'boolean', 'value': is_public}) - obj.add_attribute('abuse-confidence-score', **{'type': 'counter', 'value': abuse_confidence_score}) - obj.add_reference(request['attribute']['uuid'], "describes") + obj.add_attribute("is-whitelisted", **{"type": "boolean", "value": is_whitelisted}) + obj.add_attribute("is-tor", **{"type": "boolean", "value": is_tor}) + obj.add_attribute("is-public", **{"type": "boolean", "value": is_public}) + obj.add_attribute( + "abuse-confidence-score", + **{"type": "counter", "value": abuse_confidence_score}, + ) + obj.add_reference(request["attribute"]["uuid"], "describes") event.add_object(obj) # Avoid serialization issue event = json.loads(event.to_json()) - r['results'] = {'Object': event['Object'], 'Attribute': event['Attribute']} + r["results"] = {"Object": event["Object"], "Attribute": event["Attribute"]} return r else: try: response_json = json.loads(response.text) - if (response_json['errors']): - return {"error": "API not reachable, status code: " + str(response.status_code) + " " + str(response_json['errors'][0]['detail'])} - except: + if response_json["errors"]: + return { + "error": ( + "API not reachable, status code: " + + str(response.status_code) + + " " + + str(response_json["errors"][0]["detail"]) + ) + } + except Exception: pass return {"error": "API not reachable, status code: " + str(response.status_code)} + def introspection(): return mispattributes + def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/apiosintds.py b/misp_modules/modules/expansion/apiosintds.py index 51448ba5f..5c6799214 100644 --- a/misp_modules/modules/expansion/apiosintds.py +++ b/misp_modules/modules/expansion/apiosintds.py @@ -1,39 +1,77 @@ import json import logging -import sys import os +import sys + from apiosintDS import apiosintDS -log = logging.getLogger('apiosintDS') +log = logging.getLogger("apiosintDS") log.setLevel(logging.DEBUG) apiodbg = logging.StreamHandler(sys.stdout) apiodbg.setLevel(logging.DEBUG) -formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") apiodbg.setFormatter(formatter) log.addHandler(apiodbg) -misperrors = {'error': 'Error'} - -mispattributes = {'input': ["domain", "domain|ip", "hostname", "ip-dst", "ip-src", "ip-dst|port", "ip-src|port", "url", - "md5", "sha1", "sha256", "filename|md5", "filename|sha1", "filename|sha256"], - 'output': ["domain", "ip-dst", "url", "comment", "md5", "sha1", "sha256", "link", "text"] - } +misperrors = {"error": "Error"} + +mispattributes = { + "input": [ + "domain", + "domain|ip", + "hostname", + "ip-dst", + "ip-src", + "ip-dst|port", + "ip-src|port", + "url", + "md5", + "sha1", + "sha256", + "filename|md5", + "filename|sha1", + "filename|sha256", + ], + "output": [ + "domain", + "ip-dst", + "url", + "comment", + "md5", + "sha1", + "sha256", + "link", + "text", + ], +} moduleinfo = { - 'version': '0.2', - 'author': 'Davide Baglieri aka davidonzo', - 'description': 'On demand query API for OSINT.digitalside.it project.', - 'module-type': ['expansion', 'hover'], - 'name': 'OSINT DigitalSide', - 'logo': '', - 'requirements': ['The apiosintDS python library to query the OSINT.digitalside.it API.'], - 'features': 'The module simply queries the API of OSINT.digitalside.it with a domain, ip, url or hash attribute.\n\nThe result of the query is then parsed to extract additional hashes or urls. A module parameters also allows to parse the hashes related to the urls.\n\nFurthermore, it is possible to cache the urls and hashes collected over the last 7 days by OSINT.digitalside.it', - 'references': ['https://osint.digitalside.it/#About'], - 'input': 'A domain, ip, url or hash attribute.', - 'output': 'Hashes and urls resulting from the query to OSINT.digitalside.it', + "version": "0.2", + "author": "Davide Baglieri aka davidonzo", + "description": "On demand query API for OSINT.digitalside.it project.", + "module-type": ["expansion", "hover"], + "name": "OSINT DigitalSide", + "logo": "", + "requirements": ["The apiosintDS python library to query the OSINT.digitalside.it API."], + "features": ( + "The module simply queries the API of OSINT.digitalside.it with a domain, ip, url or hash attribute.\n\nThe" + " result of the query is then parsed to extract additional hashes or urls. A module parameters also allows to" + " parse the hashes related to the urls.\n\nFurthermore, it is possible to cache the urls and hashes collected" + " over the last 7 days by OSINT.digitalside.it" + ), + "references": ["https://osint.digitalside.it/#About"], + "input": "A domain, ip, url or hash attribute.", + "output": "Hashes and urls resulting from the query to OSINT.digitalside.it", } -moduleconfig = ['STIX2_details', 'import_related', 'cache', 'cache_directory', 'cache_timeout_h', 'local_directory'] +moduleconfig = [ + "STIX2_details", + "import_related", + "cache", + "cache_directory", + "cache_timeout_h", + "local_directory", +] def handler(q=False): @@ -41,40 +79,40 @@ def handler(q=False): return False request = json.loads(q) tosubmit = [] - if request.get('domain'): - tosubmit.append(request['domain']) - elif request.get('domain|ip'): - tosubmit.append(request['domain|ip'].split('|')[0]) - tosubmit.append(request['domain|ip'].split('|')[1]) - elif request.get('hostname'): - tosubmit.append(request['hostname']) - elif request.get('ip-dst'): - tosubmit.append(request['ip-dst']) - elif request.get('ip-src'): - tosubmit.append(request['ip-src']) - elif request.get('ip-dst|port'): - tosubmit.append(request['ip-dst|port'].split('|')[0]) - elif request.get('ip-src|port'): - tosubmit.append(request['ip-src|port'].split('|')[0]) - elif request.get('url'): - tosubmit.append(request['url']) - elif request.get('md5'): - tosubmit.append(request['md5']) - elif request.get('sha1'): - tosubmit.append(request['sha1']) - elif request.get('sha256'): - tosubmit.append(request['sha256']) - elif request.get('filename|md5'): - tosubmit.append(request['filename|md5'].split('|')[1]) - elif request.get('filename|sha1'): - tosubmit.append(request['filename|sha1'].split('|')[1]) - elif request.get('filename|sha256'): - tosubmit.append(request['filename|sha256'].split('|')[1]) + if request.get("domain"): + tosubmit.append(request["domain"]) + elif request.get("domain|ip"): + tosubmit.append(request["domain|ip"].split("|")[0]) + tosubmit.append(request["domain|ip"].split("|")[1]) + elif request.get("hostname"): + tosubmit.append(request["hostname"]) + elif request.get("ip-dst"): + tosubmit.append(request["ip-dst"]) + elif request.get("ip-src"): + tosubmit.append(request["ip-src"]) + elif request.get("ip-dst|port"): + tosubmit.append(request["ip-dst|port"].split("|")[0]) + elif request.get("ip-src|port"): + tosubmit.append(request["ip-src|port"].split("|")[0]) + elif request.get("url"): + tosubmit.append(request["url"]) + elif request.get("md5"): + tosubmit.append(request["md5"]) + elif request.get("sha1"): + tosubmit.append(request["sha1"]) + elif request.get("sha256"): + tosubmit.append(request["sha256"]) + elif request.get("filename|md5"): + tosubmit.append(request["filename|md5"].split("|")[1]) + elif request.get("filename|sha1"): + tosubmit.append(request["filename|sha1"].split("|")[1]) + elif request.get("filename|sha256"): + tosubmit.append(request["filename|sha256"].split("|")[1]) else: return False persistent = 0 - if request.get('persistent'): + if request.get("persistent"): persistent = request["persistent"] submitcache = False @@ -86,65 +124,86 @@ def handler(q=False): r = {"results": []} - if request.get('config'): + if request.get("config"): - if request['config'].get('cache') and request['config']['cache'].lower() == "yes": + if request["config"].get("cache") and request["config"]["cache"].lower() == "yes": submitcache = True - if request['config'].get('import_related') and request['config']['import_related'].lower() == "yes": + if request["config"].get("import_related") and request["config"]["import_related"].lower() == "yes": import_related = True - if request['config'].get('STIX2_details') and request['config']['STIX2_details'].lower() == "yes": + if request["config"].get("STIX2_details") and request["config"]["STIX2_details"].lower() == "yes": submit_stix = True - if request['config'].get('cache_timeout_h') and len(request['config']['cache_timeout_h']) > 0: - submitcache_timeout = int(request['config'].get('cache_timeout_h')) + if request["config"].get("cache_timeout_h") and len(request["config"]["cache_timeout_h"]) > 0: + submitcache_timeout = int(request["config"].get("cache_timeout_h")) - localdirectory = request['config'].get('local_directory') + localdirectory = request["config"].get("local_directory") if localdirectory and len(localdirectory) > 0: if os.access(localdirectory, os.R_OK): sumbit_localdirectory = localdirectory WarningMSG = "Local directory OK! Ignoring cache configuration..." log.debug(str(WarningMSG)) submitcache = False - sumbitcache_titmeout = False submitcache_directory = False else: - ErrorMSG = "Unable to read local 'Threat-Intel' directory ("+localdirectory+"). Please, check your configuration and retry." + ErrorMSG = ( + "Unable to read local 'Threat-Intel' directory (" + + localdirectory + + "). Please, check your configuration and retry." + ) log.debug(str(ErrorMSG)) - misperrors['error'] = ErrorMSG + misperrors["error"] = ErrorMSG return misperrors if submitcache: - cache_directory = request['config'].get('cache_directory') + cache_directory = request["config"].get("cache_directory") if cache_directory and len(cache_directory) > 0: if os.access(cache_directory, os.W_OK): submitcache_directory = cache_directory else: ErrorMSG = "Cache directory is not writable. Please fix it before." log.debug(str(ErrorMSG)) - misperrors['error'] = ErrorMSG + misperrors["error"] = ErrorMSG return misperrors else: - ErrorMSG = "Value for Plugin.Enrichment_apiosintds_cache_directory is empty but cache option is enabled as recommended. Please set a writable cache directory in plugin settings." + ErrorMSG = ( + "Value for Plugin.Enrichment_apiosintds_cache_directory is empty but cache option is enabled as" + " recommended. Please set a writable cache directory in plugin settings." + ) log.debug(str(ErrorMSG)) - misperrors['error'] = ErrorMSG + misperrors["error"] = ErrorMSG return misperrors else: if sumbit_localdirectory == False: - log.debug("Cache option is set to " + str(submitcache) + ". You are not using the internal cache system and this is NOT recommended!") - log.debug("Please, consider to turn on the cache setting it to 'Yes' and specifing a writable directory for the cache directory option.") + log.debug( + "Cache option is set to " + + str(submitcache) + + ". You are not using the internal cache system and this is NOT recommended!" + ) + log.debug( + "Please, consider to turn on the cache setting it to 'Yes' and specifing a writable directory for" + " the cache directory option." + ) try: - response = apiosintDS.request(entities=tosubmit, stix=submit_stix, cache=submitcache, cachedirectory=submitcache_directory, cachetimeout=submitcache_timeout, verbose=True, localdirectory=sumbit_localdirectory) + response = apiosintDS.request( + entities=tosubmit, + stix=submit_stix, + cache=submitcache, + cachedirectory=submitcache_directory, + cachetimeout=submitcache_timeout, + verbose=True, + localdirectory=sumbit_localdirectory, + ) r["results"] += apiosintParserHover(persistent, response, import_related, submit_stix) return r except Exception as e: log.exception("Could not process apiosintDS") - return {'error': str(e)} + return {"error": str(e)} def apiosintParserHover(ispersistent, response, import_related, stix): - apiosinttype = ['hash', 'ip', 'url', 'domain'] + apiosinttype = ["hash", "ip", "url", "domain"] line = "##############################################" linedot = "--------------------------------------------------------------------" linedotty = "-------------------" @@ -155,9 +214,21 @@ def apiosintParserHover(ispersistent, response, import_related, stix): if key in apiosinttype: for item in response[key]["items"]: if item["response"]: - comment = "IoC '"+item["item"] + "' found in OSINT.DigitaiSide.it repository. List file: "+response[key]["list"]["file"]+". List date: " + response[key]["list"]["date"] - commentH = "IoC '"+item["item"] + "' found in OSINT.DigitaiSide.it repository." - CommentHDate = "List file: "+response[key]["list"]["file"]+". Date list: " + response[key]["list"]["date"] + comment = ( + "IoC '" + + item["item"] + + "' found in OSINT.DigitaiSide.it repository. List file: " + + response[key]["list"]["file"] + + ". List date: " + + response[key]["list"]["date"] + ) + commentH = "IoC '" + item["item"] + "' found in OSINT.DigitaiSide.it repository." + CommentHDate = ( + "List file: " + + response[key]["list"]["file"] + + ". Date list: " + + response[key]["list"]["date"] + ) ret.append({"types": ["text"], "values": [comment]}) retHover.append({"types": ["text"], "values": [commentH]}) @@ -169,21 +240,39 @@ def apiosintParserHover(ispersistent, response, import_related, stix): headhash = "Hashes set" retHover.append({"types": ["text"], "values": [headhash]}) if "md5" in item["hashes"].keys(): - ret.append({"types": ["md5"], "values": [item["hashes"]["md5"]], "comment": "Related to: " + item["item"]}) - - strmd5 = "MD5: "+item["hashes"]["md5"] + ret.append( + { + "types": ["md5"], + "values": [item["hashes"]["md5"]], + "comment": "Related to: " + item["item"], + } + ) + + strmd5 = "MD5: " + item["hashes"]["md5"] retHover.append({"types": ["text"], "values": [strmd5]}) if "sha1" in item["hashes"].keys(): - ret.append({"types": ["sha1"], "values": [item["hashes"]["sha1"]], "comment": "Related to: " + item["item"]}) - - strsha1 = "SHA1: "+item["hashes"]["sha1"] + ret.append( + { + "types": ["sha1"], + "values": [item["hashes"]["sha1"]], + "comment": "Related to: " + item["item"], + } + ) + + strsha1 = "SHA1: " + item["hashes"]["sha1"] retHover.append({"types": ["text"], "values": [strsha1]}) if "sha256" in item["hashes"].keys(): - ret.append({"types": ["sha256"], "values": [item["hashes"]["sha256"]], "comment": "Related to: " + item["item"]}) - - strsha256 = "SHA256: "+item["hashes"]["sha256"] + ret.append( + { + "types": ["sha256"], + "values": [item["hashes"]["sha256"]], + "comment": "Related to: " + item["item"], + } + ) + + strsha256 = "SHA256: " + item["hashes"]["sha256"] retHover.append({"types": ["text"], "values": [strsha256]}) if "online_reports" in item: @@ -191,15 +280,43 @@ def apiosintParserHover(ispersistent, response, import_related, stix): retHover.append({"types": ["text"], "values": [linedot]}) retHover.append({"types": ["text"], "values": [headReports]}) onlierepor = item["online_reports"] - ret.append({"category": "External analysis", "types": ["link"], "values": [onlierepor["MISP_EVENT"]], "comment": "MISP Event related to: " + item["item"]}) - ret.append({"category": "External analysis", "types": ["link"], "values": [onlierepor["MISP_CSV"]], "comment": "MISP CSV related to: " + item["item"]}) - ret.append({"category": "External analysis", "types": ["link"], "values": [onlierepor["OSINTDS_REPORT"]], "comment": "DigitalSide report related to: " + item["item"]}) - ret.append({"category": "External analysis", "types": ["link"], "values": [onlierepor["STIX"]], "comment": "STIX2 report related to: " + item["item"]}) - - MISPEVENT = "MISP Event => "+onlierepor["MISP_EVENT"] - MISPCSV = "MISP CSV => "+onlierepor["MISP_CSV"] - OSINTDS = "DigitalSide report => "+onlierepor["OSINTDS_REPORT"] - STIX = "STIX report => "+onlierepor["STIX"] + ret.append( + { + "category": "External analysis", + "types": ["link"], + "values": [onlierepor["MISP_EVENT"]], + "comment": "MISP Event related to: " + item["item"], + } + ) + ret.append( + { + "category": "External analysis", + "types": ["link"], + "values": [onlierepor["MISP_CSV"]], + "comment": "MISP CSV related to: " + item["item"], + } + ) + ret.append( + { + "category": "External analysis", + "types": ["link"], + "values": [onlierepor["OSINTDS_REPORT"]], + "comment": "DigitalSide report related to: " + item["item"], + } + ) + ret.append( + { + "category": "External analysis", + "types": ["link"], + "values": [onlierepor["STIX"]], + "comment": "STIX2 report related to: " + item["item"], + } + ) + + MISPEVENT = "MISP Event => " + onlierepor["MISP_EVENT"] + MISPCSV = "MISP CSV => " + onlierepor["MISP_CSV"] + OSINTDS = "DigitalSide report => " + onlierepor["OSINTDS_REPORT"] + STIX = "STIX report => " + onlierepor["STIX"] retHover.append({"types": ["text"], "values": [MISPEVENT]}) retHover.append({"types": ["text"], "values": [MISPCSV]}) @@ -211,90 +328,216 @@ def apiosintParserHover(ispersistent, response, import_related, stix): retHover.append({"types": ["text"], "values": [linedot]}) headStix = "STIX2 report details" stixobj = onlierepor["STIXDETAILS"] - stxdet = "TLP:"+stixobj["tlp"]+" | Observation: "+str(stixobj["number_observed"])+" | First seen: "+stixobj["first_observed"]+" | First seen: "+stixobj["last_observed"] - ret.append({"types": ["comment"], "values": [stxdet], "comment": "STIX2 details for: " + item["item"]}) + stxdet = ( + "TLP:" + + stixobj["tlp"] + + " | Observation: " + + str(stixobj["number_observed"]) + + " | First seen: " + + stixobj["first_observed"] + + " | First seen: " + + stixobj["last_observed"] + ) + ret.append( + { + "types": ["comment"], + "values": [stxdet], + "comment": "STIX2 details for: " + item["item"], + } + ) retHover.append({"types": ["text"], "values": [headStix]}) retHover.append({"types": ["text"], "values": [stxdet]}) - if stixobj["observed_time_frame"] != False: - obstf = "Observation time frame: "+str(stixobj["observed_time_frame"]) - ret.append({"types": ["comment"], "values": [obstf], "comment": "STIX2 details for: " + item["item"]}) + obstf = "Observation time frame: " + str(stixobj["observed_time_frame"]) + ret.append( + { + "types": ["comment"], + "values": [obstf], + "comment": "STIX2 details for: " + item["item"], + } + ) retHover.append({"types": ["text"], "values": [obstf]}) filename = stixobj["filename"] - ret.append({"category": "Payload delivery", "types": ["filename"], "values": [filename], "comment": "STIX2 details for: " + item["item"]}) - - Hovefilename = "Filename: "+filename + ret.append( + { + "category": "Payload delivery", + "types": ["filename"], + "values": [filename], + "comment": "STIX2 details for: " + item["item"], + } + ) + + Hovefilename = "Filename: " + filename retHover.append({"types": ["text"], "values": [Hovefilename]}) filesize = stixobj["filesize"] - ret.append({"types": ["size-in-bytes"], "values": [filesize], "comment": "STIX2 details for: " + item["item"]}) - - Hovefilesize = "Filesize in bytes: "+str(filesize) + ret.append( + { + "types": ["size-in-bytes"], + "values": [filesize], + "comment": "STIX2 details for: " + item["item"], + } + ) + + Hovefilesize = "Filesize in bytes: " + str(filesize) retHover.append({"types": ["text"], "values": [Hovefilesize]}) filetype = stixobj["mime_type"] - ret.append({"category": "Payload delivery", "types": ["mime-type"], "values": [filetype], "comment": "STIX2 details for: " + item["item"]}) - - Hovemime = "Filetype: "+filetype + ret.append( + { + "category": "Payload delivery", + "types": ["mime-type"], + "values": [filetype], + "comment": "STIX2 details for: " + item["item"], + } + ) + + Hovemime = "Filetype: " + filetype retHover.append({"types": ["text"], "values": [Hovemime]}) if "virus_total" in stixobj: if stixobj["virus_total"] != False: - VTratio = "VirusTotal Ratio: "+str(stixobj["virus_total"]["vt_detection_ratio"]) - ret.append({"types": ["comment"], "values": [VTratio], "comment": "STIX2 details for: " + item["item"]}) + VTratio = "VirusTotal Ratio: " + str( + stixobj["virus_total"]["vt_detection_ratio"] + ) + ret.append( + { + "types": ["comment"], + "values": [VTratio], + "comment": "STIX2 details for: " + item["item"], + } + ) retHover.append({"types": ["text"], "values": [VTratio]}) VTReport = str(stixobj["virus_total"]["vt_report"]) - ret.append({"category": "External analysis", "types": ["link"], "values": [VTReport], "comment": "VirusTotal Report for: " + item["item"]}) + ret.append( + { + "category": "External analysis", + "types": ["link"], + "values": [VTReport], + "comment": "VirusTotal Report for: " + item["item"], + } + ) if import_related: if len(item["related_urls"]) > 0: retHover.append({"types": ["text"], "values": [linedot]}) - countRelated = "Related URLS count: "+str(len(item["related_urls"])) + countRelated = "Related URLS count: " + str(len(item["related_urls"])) retHover.append({"types": ["text"], "values": [countRelated]}) for urls in item["related_urls"]: if isinstance(urls, dict): itemToInclude = urls["url"] - ret.append({"types": ["url"], "values": [itemToInclude], "comment": "Download URL for "+urls["hashes"]["md5"]+". Related to: " + item["item"]}) + ret.append( + { + "types": ["url"], + "values": [itemToInclude], + "comment": ( + "Download URL for " + + urls["hashes"]["md5"] + + ". Related to: " + + item["item"] + ), + } + ) retHover.append({"types": ["text"], "values": [linedot]}) - relatedURL = "Related URL "+itemToInclude + relatedURL = "Related URL " + itemToInclude retHover.append({"types": ["text"], "values": [relatedURL]}) if "hashes" in urls.keys(): if "md5" in urls["hashes"].keys(): - ret.append({"types": ["md5"], "values": [urls["hashes"]["md5"]], "comment": "Related to: " + itemToInclude}) - - strmd5 = "MD5: "+urls["hashes"]["md5"] - retHover.append({"types": ["text"], "values": [strmd5]}) + ret.append( + { + "types": ["md5"], + "values": [urls["hashes"]["md5"]], + "comment": "Related to: " + itemToInclude, + } + ) + + strmd5 = "MD5: " + urls["hashes"]["md5"] + retHover.append( + { + "types": ["text"], + "values": [strmd5], + } + ) if "sha1" in urls["hashes"].keys(): - ret.append({"types": ["sha1"], "values": [urls["hashes"]["sha1"]], "comment": "Related to: " + itemToInclude}) - - strsha1 = "SHA1: "+urls["hashes"]["sha1"] - retHover.append({"types": ["text"], "values": [strsha1]}) + ret.append( + { + "types": ["sha1"], + "values": [urls["hashes"]["sha1"]], + "comment": "Related to: " + itemToInclude, + } + ) + + strsha1 = "SHA1: " + urls["hashes"]["sha1"] + retHover.append( + { + "types": ["text"], + "values": [strsha1], + } + ) if "sha256" in urls["hashes"].keys(): - ret.append({"types": ["sha256"], "values": [urls["hashes"]["sha256"]], "comment": "Related to: " + itemToInclude}) - - strsha256 = "SHA256: "+urls["hashes"]["sha256"] - retHover.append({"types": ["text"], "values": [strsha256]}) - + ret.append( + { + "types": ["sha256"], + "values": [urls["hashes"]["sha256"]], + "comment": "Related to: " + itemToInclude, + } + ) + + strsha256 = "SHA256: " + urls["hashes"]["sha256"] + retHover.append( + { + "types": ["text"], + "values": [strsha256], + } + ) headReports = "Online Reports (availability depends on retention)" retHover.append({"types": ["text"], "values": [linedotty]}) retHover.append({"types": ["text"], "values": [headReports]}) onlierepor = urls["online_reports"] - ret.append({"category": "External analysis", "types": ["link"], "values": [onlierepor["MISP_EVENT"]], "comment": "MISP Event related to: " + item["item"]}) - ret.append({"category": "External analysis", "types": ["link"], "values": [onlierepor["MISP_CSV"]], "comment": "MISP CSV related to: " + item["item"]}) - ret.append({"category": "External analysis", "types": ["link"], "values": [onlierepor["OSINTDS_REPORT"]], "comment": "DigitalSide report related to: " + item["item"]}) - ret.append({"category": "External analysis", "types": ["link"], "values": [onlierepor["STIX"]], "comment": "STIX2 report related to: " + item["item"]}) - - MISPEVENT = "MISP Event => "+onlierepor["MISP_EVENT"] - MISPCSV = "MISP CSV => "+onlierepor["MISP_CSV"] - OSINTDS = "DigitalSide report => "+onlierepor["OSINTDS_REPORT"] - STIX = "STIX report => "+onlierepor["STIX"] + ret.append( + { + "category": "External analysis", + "types": ["link"], + "values": [onlierepor["MISP_EVENT"]], + "comment": "MISP Event related to: " + item["item"], + } + ) + ret.append( + { + "category": "External analysis", + "types": ["link"], + "values": [onlierepor["MISP_CSV"]], + "comment": "MISP CSV related to: " + item["item"], + } + ) + ret.append( + { + "category": "External analysis", + "types": ["link"], + "values": [onlierepor["OSINTDS_REPORT"]], + "comment": "DigitalSide report related to: " + item["item"], + } + ) + ret.append( + { + "category": "External analysis", + "types": ["link"], + "values": [onlierepor["STIX"]], + "comment": "STIX2 report related to: " + item["item"], + } + ) + + MISPEVENT = "MISP Event => " + onlierepor["MISP_EVENT"] + MISPCSV = "MISP CSV => " + onlierepor["MISP_CSV"] + OSINTDS = "DigitalSide report => " + onlierepor["OSINTDS_REPORT"] + STIX = "STIX report => " + onlierepor["STIX"] retHover.append({"types": ["text"], "values": [MISPEVENT]}) retHover.append({"types": ["text"], "values": [MISPCSV]}) @@ -303,51 +546,160 @@ def apiosintParserHover(ispersistent, response, import_related, stix): if stix and onlierepor: if "STIXDETAILS" in onlierepor: - retHover.append({"types": ["text"], "values": [linedotty]}) + retHover.append( + { + "types": ["text"], + "values": [linedotty], + } + ) headStix = "STIX2 report details" stixobj = onlierepor["STIXDETAILS"] - stxdet = "TLP:"+stixobj["tlp"]+" | Observation: "+str(stixobj["number_observed"])+" | First seen: "+stixobj["first_observed"]+" | First seen: "+stixobj["last_observed"] - ret.append({"types": ["comment"], "values": [stxdet], "comment": "STIX2 details for: " + item["item"]}) - retHover.append({"types": ["text"], "values": [headStix]}) - retHover.append({"types": ["text"], "values": [stxdet]}) + stxdet = ( + "TLP:" + + stixobj["tlp"] + + " | Observation: " + + str(stixobj["number_observed"]) + + " | First seen: " + + stixobj["first_observed"] + + " | First seen: " + + stixobj["last_observed"] + ) + ret.append( + { + "types": ["comment"], + "values": [stxdet], + "comment": "STIX2 details for: " + item["item"], + } + ) + retHover.append( + { + "types": ["text"], + "values": [headStix], + } + ) + retHover.append( + { + "types": ["text"], + "values": [stxdet], + } + ) if stixobj["observed_time_frame"] != False: - obstf = "Observation time frame: "+str(stixobj["observed_time_frame"]) - ret.append({"types": ["comment"], "values": [obstf], "comment": "STIX2 details for: " + item["item"]}) - retHover.append({"types": ["text"], "values": [obstf]}) + obstf = "Observation time frame: " + str( + stixobj["observed_time_frame"] + ) + ret.append( + { + "types": ["comment"], + "values": [obstf], + "comment": "STIX2 details for: " + item["item"], + } + ) + retHover.append( + { + "types": ["text"], + "values": [obstf], + } + ) filename = stixobj["filename"] - ret.append({"category": "Payload delivery", "types": ["filename"], "values": [filename], "comment": "STIX2 details for: " + item["item"]}) - - Hovefilename = "Filename: "+filename - retHover.append({"types": ["text"], "values": [Hovefilename]}) + ret.append( + { + "category": "Payload delivery", + "types": ["filename"], + "values": [filename], + "comment": "STIX2 details for: " + item["item"], + } + ) + + Hovefilename = "Filename: " + filename + retHover.append( + { + "types": ["text"], + "values": [Hovefilename], + } + ) filesize = stixobj["filesize"] - ret.append({"types": ["size-in-bytes"], "values": [filesize], "comment": "STIX2 details for: " + item["item"]}) - - Hovefilesize = "Filesize in bytes: "+str(filesize) - retHover.append({"types": ["text"], "values": [Hovefilesize]}) + ret.append( + { + "types": ["size-in-bytes"], + "values": [filesize], + "comment": "STIX2 details for: " + item["item"], + } + ) + + Hovefilesize = "Filesize in bytes: " + str(filesize) + retHover.append( + { + "types": ["text"], + "values": [Hovefilesize], + } + ) filetype = stixobj["mime_type"] - ret.append({"category": "Payload delivery", "types": ["mime-type"], "values": [filetype], "comment": "STIX2 details for: " + item["item"]}) - - Hovemime = "Filetype: "+filetype - retHover.append({"types": ["text"], "values": [Hovemime]}) + ret.append( + { + "category": "Payload delivery", + "types": ["mime-type"], + "values": [filetype], + "comment": "STIX2 details for: " + item["item"], + } + ) + + Hovemime = "Filetype: " + filetype + retHover.append( + { + "types": ["text"], + "values": [Hovemime], + } + ) if "virus_total" in stixobj: if stixobj["virus_total"] != False: - VTratio = "VirusTotal Ratio: "+stixobj["virus_total"]["vt_detection_ratio"] - ret.append({"types": ["comment"], "values": [VTratio], "comment": "STIX2 details for: " + item["item"]}) - retHover.append({"types": ["text"], "values": [VTratio]}) + VTratio = ( + "VirusTotal Ratio: " + + stixobj["virus_total"]["vt_detection_ratio"] + ) + ret.append( + { + "types": ["comment"], + "values": [VTratio], + "comment": "STIX2 details for: " + item["item"], + } + ) + retHover.append( + { + "types": ["text"], + "values": [VTratio], + } + ) VTReport = stixobj["virus_total"]["vt_report"] - ret.append({"category": "External analysis", "types": ["link"], "values": [VTReport], "comment": "VirusTotal Report for: " + item["item"]}) + ret.append( + { + "category": "External analysis", + "types": ["link"], + "values": [VTReport], + "comment": "VirusTotal Report for: " + item["item"], + } + ) else: - ret.append({"types": ["url"], "values": [urls], "comment": "Download URL for: " + item["item"]}) - urlHover = "URL => "+urls + ret.append( + { + "types": ["url"], + "values": [urls], + "comment": "Download URL for: " + item["item"], + } + ) + urlHover = "URL => " + urls retHover.append({"types": ["text"], "values": [urlHover]}) else: - notfound = item["item"] + " IS NOT listed by OSINT.digitalside.it. Date list: " + response[key]["list"]["date"] + notfound = ( + item["item"] + + " IS NOT listed by OSINT.digitalside.it. Date list: " + + response[key]["list"]["date"] + ) ret.append({"types": ["comment"], "values": [notfound]}) retHover.append({"types": ["comment"], "values": [notfound]}) @@ -361,5 +713,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/apivoid.py b/misp_modules/modules/expansion/apivoid.py index 3410617b9..0f6006320 100755 --- a/misp_modules/modules/expansion/apivoid.py +++ b/misp_modules/modules/expansion/apivoid.py @@ -1,106 +1,157 @@ import json + import requests -from . import check_input_attribute, standard_error_message from pymisp import MISPAttribute, MISPEvent, MISPObject -misperrors = {'error': 'Error'} -mispattributes = {'input': ['domain', 'hostname', 'email', 'email-src', 'email-dst', 'email-reply-to', 'dns-soa-email', 'target-email', 'whois-registrant-email'], 'format': 'misp_standard'} +from . import check_input_attribute, standard_error_message + +misperrors = {"error": "Error"} +mispattributes = { + "input": [ + "domain", + "hostname", + "email", + "email-src", + "email-dst", + "email-reply-to", + "dns-soa-email", + "target-email", + "whois-registrant-email", + ], + "format": "misp_standard", +} moduleinfo = { - 'version': '0.2', - 'author': 'Christian Studer', - 'description': 'Module to query APIVoid with some domain attributes.', - 'module-type': ['expansion', 'hover'], - 'name': 'APIVoid', - 'logo': 'apivoid.png', - 'requirements': ['A valid APIVoid API key with enough credits to proceed 2 queries'], - 'features': 'This module takes a domain name and queries API Void to get the related DNS records and the SSL certificates. It returns then those pieces of data as MISP objects that can be added to the event.\n\nTo make it work, a valid API key and enough credits to proceed 2 queries (0.06 + 0.07 credits) are required.', - 'references': ['https://www.apivoid.com/'], - 'input': 'A domain attribute.', - 'output': 'DNS records and SSL certificates related to the domain.', + "version": "0.2", + "author": "Christian Studer", + "description": "Module to query APIVoid with some domain attributes.", + "module-type": ["expansion", "hover"], + "name": "APIVoid", + "logo": "apivoid.png", + "requirements": ["A valid APIVoid API key with enough credits to proceed 2 queries"], + "features": ( + "This module takes a domain name and queries API Void to get the related DNS records and the SSL certificates." + " It returns then those pieces of data as MISP objects that can be added to the event.\n\nTo make it work, a" + " valid API key and enough credits to proceed 2 queries (0.06 + 0.07 credits) are required." + ), + "references": ["https://www.apivoid.com/"], + "input": "A domain attribute.", + "output": "DNS records and SSL certificates related to the domain.", } -moduleconfig = ['apikey'] +moduleconfig = ["apikey"] -class APIVoidParser(): +class APIVoidParser: def __init__(self, attribute): self.misp_event = MISPEvent() self.attribute = MISPAttribute() self.attribute.from_dict(**attribute) self.misp_event.add_attribute(**self.attribute) - self.url = 'https://endpoint.apivoid.com/{}/v1/pay-as-you-go/?key={}&' + self.url = "https://endpoint.apivoid.com/{}/v1/pay-as-you-go/?key={}&" def get_results(self): - if hasattr(self, 'result'): + if hasattr(self, "result"): return self.result event = json.loads(self.misp_event.to_json()) - results = {key: event[key] for key in ('Attribute', 'Object')} - return {'results': results} + results = {key: event[key] for key in ("Attribute", "Object")} + return {"results": results} def parse_domain(self, apikey): - feature = 'dnslookup' - if requests.get(f'{self.url.format(feature, apikey)}stats').json()['credits_remained'] < 0.13: - self.result = {'error': 'You do not have enough APIVoid credits to proceed your request.'} + feature = "dnslookup" + if requests.get(f"{self.url.format(feature, apikey)}stats").json()["credits_remained"] < 0.13: + self.result = {"error": "You do not have enough APIVoid credits to proceed your request."} return - mapping = {'A': 'resolution-of', 'MX': 'mail-server-of', 'NS': 'server-name-of'} - dnslookup = requests.get(f'{self.url.format(feature, apikey)}action=dns-any&host={self.attribute.value}').json() - for item in dnslookup['data']['records']['items']: - record_type = item['type'] + mapping = {"A": "resolution-of", "MX": "mail-server-of", "NS": "server-name-of"} + dnslookup = requests.get(f"{self.url.format(feature, apikey)}action=dns-any&host={self.attribute.value}").json() + for item in dnslookup["data"]["records"]["items"]: + record_type = item["type"] try: relationship = mapping[record_type] except KeyError: continue self._handle_dns_record(item, record_type, relationship) ssl = requests.get(f'{self.url.format("sslinfo", apikey)}host={self.attribute.value}').json() - self._parse_ssl_certificate(ssl['data']['certificate']) + self._parse_ssl_certificate(ssl["data"]["certificate"]) def handle_email(self, apikey): - feature = 'emailverify' - if requests.get(f'{self.url.format(feature, apikey)}stats').json()['credits_remained'] < 0.06: - self.result = {'error': 'You do not have enough APIVoid credits to proceed your request.'} + feature = "emailverify" + if requests.get(f"{self.url.format(feature, apikey)}stats").json()["credits_remained"] < 0.06: + self.result = {"error": "You do not have enough APIVoid credits to proceed your request."} return - emaillookup = requests.get(f'{self.url.format(feature, apikey)}email={self.attribute.value}').json() - email_verification = MISPObject('apivoid-email-verification') - boolean_attributes = ['valid_format', 'suspicious_username', 'suspicious_email', 'dirty_words_username', - 'suspicious_email', 'valid_tld', 'disposable', 'has_a_records', 'has_mx_records', - 'has_spf_records', 'is_spoofable', 'dmarc_configured', 'dmarc_enforced', 'free_email', - 'russian_free_email', 'china_free_email', 'suspicious_domain', 'dirty_words_domain', - 'domain_popular', 'risky_tld', 'police_domain', 'government_domain', 'educational_domain', - 'should_block'] + emaillookup = requests.get(f"{self.url.format(feature, apikey)}email={self.attribute.value}").json() + email_verification = MISPObject("apivoid-email-verification") + boolean_attributes = [ + "valid_format", + "suspicious_username", + "suspicious_email", + "dirty_words_username", + "suspicious_email", + "valid_tld", + "disposable", + "has_a_records", + "has_mx_records", + "has_spf_records", + "is_spoofable", + "dmarc_configured", + "dmarc_enforced", + "free_email", + "russian_free_email", + "china_free_email", + "suspicious_domain", + "dirty_words_domain", + "domain_popular", + "risky_tld", + "police_domain", + "government_domain", + "educational_domain", + "should_block", + ] for boolean_attribute in boolean_attributes: - email_verification.add_attribute(boolean_attribute, - **{'type': 'boolean', 'value': emaillookup['data'][boolean_attribute]}) - email_verification.add_attribute('email', **{'type': 'email', 'value': emaillookup['data']['email']}) - email_verification.add_attribute('username', **{'type': 'text', 'value': emaillookup['data']['username']}) - email_verification.add_attribute('role_address', - **{'type': 'boolean', 'value': emaillookup['data']['role_address']}) - email_verification.add_attribute('domain', **{'type': 'domain', 'value': emaillookup['data']['domain']}) - email_verification.add_attribute('score', **{'type': 'float', 'value': emaillookup['data']['score']}) - email_verification.add_reference(self.attribute['uuid'], 'related-to') + email_verification.add_attribute( + boolean_attribute, + **{"type": "boolean", "value": emaillookup["data"][boolean_attribute]}, + ) + email_verification.add_attribute("email", **{"type": "email", "value": emaillookup["data"]["email"]}) + email_verification.add_attribute("username", **{"type": "text", "value": emaillookup["data"]["username"]}) + email_verification.add_attribute( + "role_address", + **{"type": "boolean", "value": emaillookup["data"]["role_address"]}, + ) + email_verification.add_attribute("domain", **{"type": "domain", "value": emaillookup["data"]["domain"]}) + email_verification.add_attribute("score", **{"type": "float", "value": emaillookup["data"]["score"]}) + email_verification.add_reference(self.attribute["uuid"], "related-to") self.misp_event.add_object(email_verification) def _handle_dns_record(self, item, record_type, relationship): - dns_record = MISPObject('dns-record') - dns_record.add_attribute('queried-domain', type='domain', value=item['host']) - attribute_type, feature = ('ip-dst', 'ip') if record_type == 'A' else ('domain', 'target') - dns_record.add_attribute(f'{record_type.lower()}-record', type=attribute_type, value=item[feature]) + dns_record = MISPObject("dns-record") + dns_record.add_attribute("queried-domain", type="domain", value=item["host"]) + attribute_type, feature = ("ip-dst", "ip") if record_type == "A" else ("domain", "target") + dns_record.add_attribute(f"{record_type.lower()}-record", type=attribute_type, value=item[feature]) dns_record.add_reference(self.attribute.uuid, relationship) self.misp_event.add_object(**dns_record) def _parse_ssl_certificate(self, certificate): - x509 = MISPObject('x509') - fingerprint = 'x509-fingerprint-sha1' - x509.add_attribute(fingerprint, type=fingerprint, value=certificate['fingerprint']) - x509_mapping = {'subject': {'name': ('text', 'subject')}, - 'issuer': {'common_name': ('text', 'issuer')}, - 'signature': {'serial': ('text', 'serial-number')}, - 'validity': {'valid_from': ('datetime', 'validity-not-before'), - 'valid_to': ('datetime', 'validity-not-after')}} - certificate = certificate['details'] + x509 = MISPObject("x509") + fingerprint = "x509-fingerprint-sha1" + x509.add_attribute(fingerprint, type=fingerprint, value=certificate["fingerprint"]) + x509_mapping = { + "subject": {"name": ("text", "subject")}, + "issuer": {"common_name": ("text", "issuer")}, + "signature": {"serial": ("text", "serial-number")}, + "validity": { + "valid_from": ("datetime", "validity-not-before"), + "valid_to": ("datetime", "validity-not-after"), + }, + } + certificate = certificate["details"] for feature, subfeatures in x509_mapping.items(): for subfeature, mapping in subfeatures.items(): attribute_type, relation = mapping - x509.add_attribute(relation, type=attribute_type, value=certificate[feature][subfeature]) - x509.add_reference(self.attribute.uuid, 'seen-by') + x509.add_attribute( + relation, + type=attribute_type, + value=certificate[feature][subfeature], + ) + x509.add_reference(self.attribute.uuid, "seen-by") self.misp_event.add_object(**x509) @@ -108,16 +159,16 @@ def handler(q=False): if q is False: return False request = json.loads(q) - if not request.get('config', {}).get('apikey'): - return {'error': 'An API key for APIVoid is required.'} - if not request.get('attribute') or not check_input_attribute(request['attribute']): - return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} - attribute = request['attribute'] - if attribute['type'] not in mispattributes['input']: - return {'error': 'Unsupported attribute type.'} - apikey = request['config']['apikey'] + if not request.get("config", {}).get("apikey"): + return {"error": "An API key for APIVoid is required."} + if not request.get("attribute") or not check_input_attribute(request["attribute"]): + return {"error": f"{standard_error_message}, which should contain at least a type, a value and an uuid."} + attribute = request["attribute"] + if attribute["type"] not in mispattributes["input"]: + return {"error": "Unsupported attribute type."} + apikey = request["config"]["apikey"] apivoid_parser = APIVoidParser(attribute) - if attribute['type'] in ['domain', 'hostname']: + if attribute["type"] in ["domain", "hostname"]: apivoid_parser.parse_domain(apikey) else: apivoid_parser.handle_email(apikey) @@ -129,5 +180,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/assemblyline_query.py b/misp_modules/modules/expansion/assemblyline_query.py index 3c5867c37..ef071fbd0 100644 --- a/misp_modules/modules/expansion/assemblyline_query.py +++ b/misp_modules/modules/expansion/assemblyline_query.py @@ -1,81 +1,104 @@ # -*- coding: utf-8 -*- import json -from . import check_input_attribute, standard_error_message -from assemblyline_client import Client, ClientError from collections import defaultdict + +from assemblyline_client import Client, ClientError from pymisp import MISPAttribute, MISPEvent, MISPObject -misperrors = {'error': 'Error'} -mispattributes = {'input': ['link'], 'format': 'misp_standard'} +from . import check_input_attribute, standard_error_message + +misperrors = {"error": "Error"} +mispattributes = {"input": ["link"], "format": "misp_standard"} moduleinfo = { - 'version': '1', - 'author': 'Christian Studer', - 'description': 'A module tu query the AssemblyLine API with a submission ID to get the submission report and parse it.', - 'module-type': ['expansion'], - 'name': 'AssemblyLine Query', - 'logo': 'assemblyline.png', - 'requirements': ['assemblyline_client: Python library to query the AssemblyLine rest API.'], - 'features': 'The module requires the address of the AssemblyLine server you want to query as well as your credentials used for this instance. Credentials include the used-ID and an API key or the password associated to the user-ID.\n\nThe submission ID extracted from the submission link is then used to query AssemblyLine and get the full submission report. This report is parsed to extract file objects and the associated IPs, domains or URLs the files are connecting to.\n\nSome more data may be parsed in the future.', - 'references': ['https://www.cyber.gc.ca/en/assemblyline'], - 'input': 'Link of an AssemblyLine submission report.', - 'output': 'MISP attributes & objects parsed from the AssemblyLine submission.', + "version": "1", + "author": "Christian Studer", + "description": ( + "A module tu query the AssemblyLine API with a submission ID to get the submission report and parse it." + ), + "module-type": ["expansion"], + "name": "AssemblyLine Query", + "logo": "assemblyline.png", + "requirements": ["assemblyline_client: Python library to query the AssemblyLine rest API."], + "features": ( + "The module requires the address of the AssemblyLine server you want to query as well as your credentials used" + " for this instance. Credentials include the used-ID and an API key or the password associated to the" + " user-ID.\n\nThe submission ID extracted from the submission link is then used to query AssemblyLine and get" + " the full submission report. This report is parsed to extract file objects and the associated IPs, domains or" + " URLs the files are connecting to.\n\nSome more data may be parsed in the future." + ), + "references": ["https://www.cyber.gc.ca/en/assemblyline"], + "input": "Link of an AssemblyLine submission report.", + "output": "MISP attributes & objects parsed from the AssemblyLine submission.", } moduleconfig = ["apiurl", "user_id", "apikey", "password", "verifyssl"] -class AssemblyLineParser(): +class AssemblyLineParser: def __init__(self): self.misp_event = MISPEvent() self.results = {} - self.attribute = {'to_ids': True} - self._results_mapping = {'NET_DOMAIN_NAME': 'domain', 'NET_FULL_URI': 'url', - 'NET_IP': 'ip-dst'} - self._file_mapping = {'entropy': {'type': 'float', 'object_relation': 'entropy'}, - 'md5': {'type': 'md5', 'object_relation': 'md5'}, - 'mime': {'type': 'mime-type', 'object_relation': 'mimetype'}, - 'sha1': {'type': 'sha1', 'object_relation': 'sha1'}, - 'sha256': {'type': 'sha256', 'object_relation': 'sha256'}, - 'size': {'type': 'size-in-bytes', 'object_relation': 'size-in-bytes'}, - 'ssdeep': {'type': 'ssdeep', 'object_relation': 'ssdeep'}} + self.attribute = {"to_ids": True} + self._results_mapping = { + "NET_DOMAIN_NAME": "domain", + "NET_FULL_URI": "url", + "NET_IP": "ip-dst", + } + self._file_mapping = { + "entropy": {"type": "float", "object_relation": "entropy"}, + "md5": {"type": "md5", "object_relation": "md5"}, + "mime": {"type": "mime-type", "object_relation": "mimetype"}, + "sha1": {"type": "sha1", "object_relation": "sha1"}, + "sha256": {"type": "sha256", "object_relation": "sha256"}, + "size": {"type": "size-in-bytes", "object_relation": "size-in-bytes"}, + "ssdeep": {"type": "ssdeep", "object_relation": "ssdeep"}, + } def get_submission(self, attribute, client): - sid = attribute['value'].split('=')[-1] + sid = attribute["value"].split("=")[-1] try: if not client.submission.is_completed(sid): - self.results['error'] = 'Submission not completed, please try again later.' + self.results["error"] = "Submission not completed, please try again later." return except Exception as e: - self.results['error'] = f'Something went wrong while trying to check if the submission in AssemblyLine is completed: {e.__str__()}' + self.results["error"] = ( + "Something went wrong while trying to check if the submission in AssemblyLine is completed:" + f" {e.__str__()}" + ) return try: submission = client.submission.full(sid) except Exception as e: - self.results['error'] = f"Something went wrong while getting the submission from AssemblyLine: {e.__str__()}" + self.results["error"] = ( + f"Something went wrong while getting the submission from AssemblyLine: {e.__str__()}" + ) return self._parse_report(submission) def finalize_results(self): - if 'error' in self.results: + if "error" in self.results: return self.results event = json.loads(self.misp_event.to_json()) - results = {key: event[key] for key in ('Attribute', 'Object', 'Tag') if (key in event and event[key])} - return {'results': results} + results = {key: event[key] for key in ("Attribute", "Object", "Tag") if (key in event and event[key])} + return {"results": results} def _create_attribute(self, result, attribute_type): attribute = MISPAttribute() - attribute.from_dict(type=attribute_type, value=result['value'], **self.attribute) - if result['classification'] != 'UNCLASSIFIED': - attribute.add_tag(result['classification'].lower()) + attribute.from_dict(type=attribute_type, value=result["value"], **self.attribute) + if result["classification"] != "UNCLASSIFIED": + attribute.add_tag(result["classification"].lower()) self.misp_event.add_attribute(**attribute) - return {'referenced_uuid': attribute.uuid, 'relationship_type': '-'.join(result['context'].lower().split(' '))} + return { + "referenced_uuid": attribute.uuid, + "relationship_type": "-".join(result["context"].lower().split(" ")), + } def _create_file_object(self, file_info): - file_object = MISPObject('file') - filename_attribute = {'type': 'filename'} + file_object = MISPObject("file") + filename_attribute = {"type": "filename"} filename_attribute.update(self.attribute) - if file_info['classification'] != "UNCLASSIFIED": - tag = {'Tag': [{'name': file_info['classification'].lower()}]} + if file_info["classification"] != "UNCLASSIFIED": + tag = {"Tag": [{"name": file_info["classification"].lower()}]} filename_attribute.update(tag) for feature, attribute in self._file_mapping.items(): attribute.update(tag) @@ -89,33 +112,33 @@ def _create_file_object(self, file_info): def _get_results(submission_results): results = defaultdict(list) for k, values in submission_results.items(): - h = k.split('.')[0] - for t in values['result']['tags']: - if t['context'] is not None: + h = k.split(".")[0] + for t in values["result"]["tags"]: + if t["context"] is not None: results[h].append(t) return results def _get_scores(self, file_tree): scores = {} for h, f in file_tree.items(): - score = f['score'] + score = f["score"] if score > 0: - scores[h] = {'name': f['name'], 'score': score} - if f['children']: - scores.update(self._get_scores(f['children'])) + scores[h] = {"name": f["name"], "score": score} + if f["children"]: + scores.update(self._get_scores(f["children"])) return scores def _parse_report(self, submission): - if submission['classification'] != 'UNCLASSIFIED': - self.misp_event.add_tag(submission['classification'].lower()) - filtered_results = self._get_results(submission['results']) - scores = self._get_scores(submission['file_tree']) + if submission["classification"] != "UNCLASSIFIED": + self.misp_event.add_tag(submission["classification"].lower()) + filtered_results = self._get_results(submission["results"]) + scores = self._get_scores(submission["file_tree"]) for h, results in filtered_results.items(): if h in scores: - attribute, file_object = self._create_file_object(submission['file_infos'][h]) + attribute, file_object = self._create_file_object(submission["file_infos"][h]) print(file_object) - for filename in scores[h]['name']: - file_object.add_attribute('filename', value=filename, **attribute) + for filename in scores[h]["name"]: + file_object.add_attribute("filename", value=filename, **attribute) for reference in self._parse_results(results): file_object.add_reference(**reference) self.misp_event.add_object(**file_object) @@ -124,7 +147,7 @@ def _parse_results(self, results): references = [] for result in results: try: - attribute_type = self._results_mapping[result['type']] + attribute_type = self._results_mapping[result["type"]] except KeyError: continue references.append(self._create_attribute(result, attribute_type)) @@ -133,16 +156,16 @@ def _parse_results(self, results): def parse_config(apiurl, user_id, config): error = {"error": "Please provide your AssemblyLine API key or Password."} - if config.get('apikey'): + if config.get("apikey"): try: - return Client(apiurl, apikey=(user_id, config['apikey']), verify=config['verifyssl']) + return Client(apiurl, apikey=(user_id, config["apikey"]), verify=config["verifyssl"]) except ClientError as e: - error['error'] = f'Error while initiating a connection with AssemblyLine: {e.__str__()}' - if config.get('password'): + error["error"] = f"Error while initiating a connection with AssemblyLine: {e.__str__()}" + if config.get("password"): try: - return Client(apiurl, auth=(user_id, config['password'])) + return Client(apiurl, auth=(user_id, config["password"])) except ClientError as e: - error['error'] = f'Error while initiating a connection with AssemblyLine: {e.__str__()}' + error["error"] = f"Error while initiating a connection with AssemblyLine: {e.__str__()}" return error @@ -150,23 +173,23 @@ def handler(q=False): if q is False: return False request = json.loads(q) - if not request.get('attribute') or not check_input_attribute(request['attribute']): - return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} - if request['attribute']['type'] not in mispattributes['input']: - return {'error': 'Unsupported attribute type.'} - if not request.get('config'): + if not request.get("attribute") or not check_input_attribute(request["attribute"]): + return {"error": f"{standard_error_message}, which should contain at least a type, a value and an uuid."} + if request["attribute"]["type"] not in mispattributes["input"]: + return {"error": "Unsupported attribute type."} + if not request.get("config"): return {"error": "Missing configuration."} - if not request['config'].get('apiurl'): + if not request["config"].get("apiurl"): return {"error": "No AssemblyLine server address provided."} - apiurl = request['config']['apiurl'] - if not request['config'].get('user_id'): + apiurl = request["config"]["apiurl"] + if not request["config"].get("user_id"): return {"error": "Please provide your AssemblyLine User ID."} - user_id = request['config']['user_id'] - client = parse_config(apiurl, user_id, request['config']) + user_id = request["config"]["user_id"] + client = parse_config(apiurl, user_id, request["config"]) if isinstance(client, dict): return client assemblyline_parser = AssemblyLineParser() - assemblyline_parser.get_submission(request['attribute'], client) + assemblyline_parser.get_submission(request["attribute"], client) return assemblyline_parser.finalize_results() @@ -175,5 +198,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/assemblyline_submit.py b/misp_modules/modules/expansion/assemblyline_submit.py index 9d3681c5a..d7e2143e0 100644 --- a/misp_modules/modules/expansion/assemblyline_submit.py +++ b/misp_modules/modules/expansion/assemblyline_submit.py @@ -1,40 +1,45 @@ # -*- coding: utf-8 -*- import json - -from assemblyline_client import Client, ClientError from urllib.parse import urljoin +from assemblyline_client import Client, ClientError moduleinfo = { - 'version': 1, - 'author': 'Christian Studer', - 'module-type': ['expansion'], - 'name': 'AssemblyLine Submit', - 'description': 'A module to submit samples and URLs to AssemblyLine for advanced analysis, and return the link of the submission.', - 'logo': 'assemblyline.png', - 'requirements': ['assemblyline_client: Python library to query the AssemblyLine rest API.'], - 'features': 'The module requires the address of the AssemblyLine server you want to query as well as your credentials used for this instance. Credentials include the user-ID and an API key or the password associated to the user-ID.\n\nIf the sample or url is correctly submitted, you get then the link of the submission.', - 'references': ['https://www.cyber.gc.ca/en/assemblyline'], - 'input': 'Sample, or url to submit to AssemblyLine.', - 'output': 'Link of the report generated in AssemblyLine.', + "version": 1, + "author": "Christian Studer", + "module-type": ["expansion"], + "name": "AssemblyLine Submit", + "description": ( + "A module to submit samples and URLs to AssemblyLine for advanced analysis, and return the link of the" + " submission." + ), + "logo": "assemblyline.png", + "requirements": ["assemblyline_client: Python library to query the AssemblyLine rest API."], + "features": ( + "The module requires the address of the AssemblyLine server you want to query as well as your credentials used" + " for this instance. Credentials include the user-ID and an API key or the password associated to the" + " user-ID.\n\nIf the sample or url is correctly submitted, you get then the link of the submission." + ), + "references": ["https://www.cyber.gc.ca/en/assemblyline"], + "input": "Sample, or url to submit to AssemblyLine.", + "output": "Link of the report generated in AssemblyLine.", } moduleconfig = ["apiurl", "user_id", "apikey", "password", "verifyssl"] -mispattributes = {"input": ["attachment", "malware-sample", "url"], - "output": ["link"]} +mispattributes = {"input": ["attachment", "malware-sample", "url"], "output": ["link"]} def parse_config(apiurl, user_id, config): error = {"error": "Please provide your AssemblyLine API key or Password."} - if config.get('apikey'): + if config.get("apikey"): try: - return Client(apiurl, apikey=(user_id, config['apikey']), verify=config['verifyssl']) + return Client(apiurl, apikey=(user_id, config["apikey"]), verify=config["verifyssl"]) except ClientError as e: - error['error'] = f'Error while initiating a connection with AssemblyLine: {e.__str__()}' - if config.get('password'): + error["error"] = f"Error while initiating a connection with AssemblyLine: {e.__str__()}" + if config.get("password"): try: - return Client(apiurl, auth=(user_id, config['password']), verify=config['verifyssl']) + return Client(apiurl, auth=(user_id, config["password"]), verify=config["verifyssl"]) except ClientError as e: - error['error'] = f'Error while initiating a connection with AssemblyLine: {e.__str__()}' + error["error"] = f"Error while initiating a connection with AssemblyLine: {e.__str__()}" return error @@ -42,15 +47,15 @@ def submit_content(client, filename, data): try: return client.submit(fname=filename, contents=data.encode()) except Exception as e: - return {'error': f'Error while submitting content to AssemblyLine: {e.__str__()}'} + return {"error": f"Error while submitting content to AssemblyLine: {e.__str__()}"} def submit_request(client, request): - if 'attachment' in request: - return submit_content(client, request['attachment'], request['data']) - if 'malware-sample' in request: - return submit_content(client, request['malware-sample'].split('|')[0], request['data']) - for feature in ('url', 'domain'): + if "attachment" in request: + return submit_content(client, request["attachment"], request["data"]) + if "malware-sample" in request: + return submit_content(client, request["malware-sample"].split("|")[0], request["data"]) + for feature in ("url", "domain"): if feature in request: return submit_url(client, request[feature]) return {"error": "No valid attribute type for this module has been provided."} @@ -60,34 +65,36 @@ def submit_url(client, url): try: return client.submit(url=url) except Exception as e: - return {'error': f'Error while submitting url to AssemblyLine: {e.__str__()}'} + return {"error": f"Error while submitting url to AssemblyLine: {e.__str__()}"} def handler(q=False): if q is False: return q request = json.loads(q) - if not request.get('config'): + if not request.get("config"): return {"error": "Missing configuration."} - if not request['config'].get('apiurl'): + if not request["config"].get("apiurl"): return {"error": "No AssemblyLine server address provided."} - apiurl = request['config']['apiurl'] - if not request['config'].get('user_id'): + apiurl = request["config"]["apiurl"] + if not request["config"].get("user_id"): return {"error": "Please provide your AssemblyLine User ID."} - user_id = request['config']['user_id'] - client = parse_config(apiurl, user_id, request['config']) + user_id = request["config"]["user_id"] + client = parse_config(apiurl, user_id, request["config"]) if isinstance(client, dict): return client submission = submit_request(client, request) - if 'error' in submission: + if "error" in submission: return submission - sid = submission['submission']['sid'] + sid = submission["submission"]["sid"] return { - "results": [{ - "types": "link", - "categories": "External analysis", - "values": urljoin(apiurl, f'submission_detail.html?sid={sid}') - }] + "results": [ + { + "types": "link", + "categories": "External analysis", + "values": urljoin(apiurl, f"submission_detail.html?sid={sid}"), + } + ] } diff --git a/misp_modules/modules/expansion/backscatter_io.py b/misp_modules/modules/expansion/backscatter_io.py index d226f503b..65f5ea460 100644 --- a/misp_modules/modules/expansion/backscatter_io.py +++ b/misp_modules/modules/expansion/backscatter_io.py @@ -1,54 +1,59 @@ # -*- coding: utf-8 -*- """Backscatter.io Module.""" import json -try: - from backscatter import Backscatter -except ImportError: - print("Backscatter.io library not installed.") -misperrors = {'error': 'Error'} -mispattributes = {'input': ['ip-src', 'ip-dst'], 'output': ['freetext']} +from backscatter import Backscatter + +misperrors = {"error": "Error"} +mispattributes = {"input": ["ip-src", "ip-dst"], "output": ["freetext"]} moduleinfo = { - 'version': '1', - 'author': 'brandon@backscatter.io', - 'description': 'Backscatter.io module to bring mass-scanning observations into MISP.', - 'module-type': ['expansion', 'hover'], - 'name': 'Backscatter.io', - 'logo': 'backscatter_io.png', - 'requirements': ['backscatter python library'], - 'features': 'The module takes a source or destination IP address as input and displays the information known by backscatter.io.', - 'references': ['https://pypi.org/project/backscatter/'], - 'input': 'IP addresses.', - 'output': 'Text containing a history of the IP addresses especially on scanning based on backscatter.io information .', + "version": "1", + "author": "brandon@backscatter.io", + "description": "Backscatter.io module to bring mass-scanning observations into MISP.", + "module-type": ["expansion", "hover"], + "name": "Backscatter.io", + "logo": "backscatter_io.png", + "requirements": ["backscatter python library"], + "features": ( + "The module takes a source or destination IP address as input and displays the information known by" + " backscatter.io." + ), + "references": ["https://pypi.org/project/backscatter/"], + "input": "IP addresses.", + "output": ( + "Text containing a history of the IP addresses especially on scanning based on backscatter.io information ." + ), } -moduleconfig = ['api_key'] +moduleconfig = ["api_key"] query_playbook = [ - {'inputs': ['ip-src', 'ip-dst'], - 'services': ['observations', 'enrichment'], - 'name': 'generic'} + { + "inputs": ["ip-src", "ip-dst"], + "services": ["observations", "enrichment"], + "name": "generic", + } ] def check_query(request): """Check the incoming request for a valid configuration.""" - output = {'success': False} - config = request.get('config', None) + output = {"success": False} + config = request.get("config", None) if not config: - misperrors['error'] = "Configuration is missing from the request." + misperrors["error"] = "Configuration is missing from the request." return output for item in moduleconfig: if config.get(item, None): continue - misperrors['error'] = "Backscatter.io authentication is missing." + misperrors["error"] = "Backscatter.io authentication is missing." return output - if not request.get('ip-src') and request.get('ip-dst'): - misperrors['error'] = "Unsupported attributes type." + if not request.get("ip-src") and request.get("ip-dst"): + misperrors["error"] = "Unsupported attributes type." return output - profile = {'success': True, 'config': config, 'playbook': 'generic'} - if 'ip-src' in request: - profile.update({'value': request.get('ip-src')}) + profile = {"success": True, "config": config, "playbook": "generic"} + if "ip-src" in request: + profile.update({"value": request.get("ip-src")}) else: - profile.update({'value': request.get('ip-dst')}) + profile.update({"value": request.get("ip-dst")}) return profile @@ -58,18 +63,18 @@ def handler(q=False): return q request = json.loads(q) checks = check_query(request) - if not checks['success']: + if not checks["success"]: return misperrors try: - bs = Backscatter(checks['config']['api_key']) - response = bs.get_observations(query=checks['value'], query_type='ip') - if not response['success']: - misperrors['error'] = '%s: %s' % (response['error'], response['message']) + bs = Backscatter(checks["config"]["api_key"]) + response = bs.get_observations(query=checks["value"], query_type="ip") + if not response["success"]: + misperrors["error"] = "%s: %s" % (response["error"], response["message"]) return misperrors - output = {'results': [{'types': mispattributes['output'], 'values': [str(response)]}]} + output = {"results": [{"types": mispattributes["output"], "values": [str(response)]}]} except Exception as e: - misperrors['error'] = str(e) + misperrors["error"] = str(e) return misperrors return output @@ -80,5 +85,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/btc_scam_check.py b/misp_modules/modules/expansion/btc_scam_check.py index 8b577da49..9e0d11555 100644 --- a/misp_modules/modules/expansion/btc_scam_check.py +++ b/misp_modules/modules/expansion/btc_scam_check.py @@ -1,48 +1,51 @@ import json -import sys - -try: - from dns.resolver import Resolver, NXDOMAIN - from dns.name import LabelTooLong - resolver = Resolver() - resolver.timeout = 1 - resolver.lifetime = 1 -except ImportError: - sys.exit("dnspython3 in missing. use 'pip install dnspython3' to install it.") - -misperrors = {'error': 'Error'} -mispattributes = {'input': ['btc'], 'output': ['text']} + +from dns.name import LabelTooLong +from dns.resolver import NXDOMAIN, Resolver + +resolver = Resolver() +resolver.timeout = 1 +resolver.lifetime = 1 + + +misperrors = {"error": "Error"} +mispattributes = {"input": ["btc"], "output": ["text"]} moduleinfo = { - 'version': '0.1', - 'author': 'Christian Studer', - 'description': 'An expansion hover module to query a special dns blacklist to check if a bitcoin address has been abused.', - 'module-type': ['hover'], - 'name': 'BTC Scam Check', - 'logo': 'bitcoin.png', - 'requirements': ['dnspython3: dns python library'], - 'features': 'The module queries a dns blacklist directly with the bitcoin address and get a response if the address has been abused.', - 'references': ['https://btcblack.it/'], - 'input': 'btc address attribute.', - 'output': 'Text to indicate if the BTC address has been abused.', + "version": "0.1", + "author": "Christian Studer", + "description": ( + "An expansion hover module to query a special dns blacklist to check if a bitcoin address has been abused." + ), + "module-type": ["hover"], + "name": "BTC Scam Check", + "logo": "bitcoin.png", + "requirements": ["dnspython3: dns python library"], + "features": ( + "The module queries a dns blacklist directly with the bitcoin address and get a response if the address has" + " been abused." + ), + "references": ["https://btcblack.it/"], + "input": "btc address attribute.", + "output": "Text to indicate if the BTC address has been abused.", } moduleconfig = [] -url = 'bl.btcblack.it' +url = "bl.btcblack.it" def handler(q=False): if q is False: return False request = json.loads(q) - btc = request['btc'] + btc = request["btc"] query = f"{btc}.{url}" try: - result = ' - '.join([str(r) for r in resolver.resolve(query, 'TXT')])[1:-1] + result = " - ".join([str(r) for r in resolver.resolve(query, "TXT")])[1:-1] except NXDOMAIN: result = f"{btc} is not known as a scam address." except LabelTooLong: result = f"{btc} is probably not a valid BTC address." - return {'results': [{'types': mispattributes['output'], 'values': result}]} + return {"results": [{"types": mispattributes["output"], "values": result}]} def introspection(): @@ -50,5 +53,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/btc_steroids.py b/misp_modules/modules/expansion/btc_steroids.py index 899c64b03..88f293127 100755 --- a/misp_modules/modules/expansion/btc_steroids.py +++ b/misp_modules/modules/expansion/btc_steroids.py @@ -1,31 +1,32 @@ import json -import requests import time -misperrors = {'error': 'Error'} -mispattributes = {'input': ['btc'], 'output': ['text']} +import requests + +misperrors = {"error": "Error"} +mispattributes = {"input": ["btc"], "output": ["text"]} moduleinfo = { - 'version': '0.1', - 'author': 'Sascha Rommelfangen', - 'description': 'An expansion hover module to get a blockchain balance from a BTC address in MISP.', - 'module-type': ['hover'], - 'name': 'BTC Steroids', - 'logo': 'bitcoin.png', - 'requirements': [], - 'features': '', - 'references': [], - 'input': 'btc address attribute.', - 'output': 'Text to describe the blockchain balance and the transactions related to the btc address in input.', + "version": "0.1", + "author": "Sascha Rommelfangen", + "description": "An expansion hover module to get a blockchain balance from a BTC address in MISP.", + "module-type": ["hover"], + "name": "BTC Steroids", + "logo": "bitcoin.png", + "requirements": [], + "features": "", + "references": [], + "input": "btc address attribute.", + "output": "Text to describe the blockchain balance and the transactions related to the btc address in input.", } moduleconfig = [] -blockchain_firstseen = 'https://blockchain.info/q/addressfirstseen/' -blockchain_balance = 'https://blockchain.info/q/addressbalance/' -blockchain_totalreceived = 'https://blockchain.info/q/getreceivedbyaddress/' -blockchain_all = 'https://blockchain.info/rawaddr/{}?filter=5{}' -converter = 'https://min-api.cryptocompare.com/data/pricehistorical?fsym=BTC&tsyms=USD,EUR&ts={}' -converter_rls = 'https://min-api.cryptocompare.com/stats/rate/limit' +blockchain_firstseen = "https://blockchain.info/q/addressfirstseen/" +blockchain_balance = "https://blockchain.info/q/addressbalance/" +blockchain_totalreceived = "https://blockchain.info/q/getreceivedbyaddress/" +blockchain_all = "https://blockchain.info/rawaddr/{}?filter=5{}" +converter = "https://min-api.cryptocompare.com/data/pricehistorical?fsym=BTC&tsyms=USD,EUR&ts={}" +converter_rls = "https://min-api.cryptocompare.com/stats/rate/limit" result_text = "" g_rate_limit = 300 start_time = 0 @@ -36,8 +37,8 @@ def get_consumption(output=False): try: req = requests.get(converter_rls) jreq = req.json() - minute = str(jreq['Data']['calls_left']['minute']) - hour = str(jreq['Data']['calls_left']['hour']) + minute = str(jreq["Data"]["calls_left"]["minute"]) + hour = str(jreq["Data"]["calls_left"]["hour"]) except Exception: minute = str(-1) hour = str(-1) @@ -51,7 +52,7 @@ def convert(btc, timestamp): global start_time global now global conversion_rates - date = time.strftime('%Y-%m-%d', time.localtime(timestamp)) + date = time.strftime("%Y-%m-%d", time.localtime(timestamp)) # Lookup conversion rates in the cache: if date in conversion_rates: (usd, eur) = conversion_rates[date] @@ -77,8 +78,8 @@ def convert(btc, timestamp): try: req = requests.get(converter.format(timestamp)) jreq = req.json() - usd = jreq['BTC']['USD'] - eur = jreq['BTC']['EUR'] + usd = jreq["BTC"]["USD"] + eur = jreq["BTC"]["EUR"] # Since we have the rates, store them in the cache conversion_rates[date] = (usd, eur) except Exception as ex: @@ -107,11 +108,11 @@ def handler(q=False): request = json.loads(q) click = False # This means the magnifying glass has been clicked - if request.get('persistent') == 1: + if request.get("persistent") == 1: click = True # Otherwise the attribute was only hovered over - if request.get('btc'): - btc = request['btc'] + if request.get("btc"): + btc = request["btc"] else: return False mprint("\nAddress:\t" + btc) @@ -122,23 +123,14 @@ def handler(q=False): # print(e) print(req.text) result_text = "Not a valid BTC address" - r = { - 'results': [ - { - 'types': ['text'], - 'values':[ - str(result_text) - ] - } - ] - } + r = {"results": [{"types": ["text"], "values": [str(result_text)]}]} return r - n_tx = jreq['n_tx'] - balance = float(jreq['final_balance'] / 100000000) - rcvd = float(jreq['total_received'] / 100000000) - sent = float(jreq['total_sent'] / 100000000) - output = 'Balance:\t{0:.10f} BTC (+{1:.10f} BTC / -{2:.10f} BTC)' + n_tx = jreq["n_tx"] + balance = float(jreq["final_balance"] / 100000000) + rcvd = float(jreq["total_received"] / 100000000) + sent = float(jreq["total_sent"] / 100000000) + output = "Balance:\t{0:.10f} BTC (+{1:.10f} BTC / -{2:.10f} BTC)" mprint(output.format(balance, rcvd, sent)) if click is False: mprint("Transactions:\t" + str(n_tx) + "\t (previewing up to 5 most recent)") @@ -167,61 +159,74 @@ def handler(q=False): time.sleep(3) req = requests.get(blockchain_all.format(btc, "&limit=50&offset={}".format(i))) jreq = req.json() - if jreq['txs']: - for transactions in jreq['txs']: + if jreq["txs"]: + for transactions in jreq["txs"]: sum = 0 sum_counter = 0 - for tx in transactions['inputs']: - script_old = tx['script'] + for tx in transactions["inputs"]: + script_old = tx["script"] try: - addr_in = tx['prev_out']['addr'] + addr_in = tx["prev_out"]["addr"] except KeyError: addr_in = None try: - prev_out = tx['prev_out']['value'] + prev_out = tx["prev_out"]["value"] except KeyError: prev_out = None if prev_out != 0 and addr_in == btc: - datetime = time.strftime("%d %b %Y %H:%M:%S %Z", time.localtime(int(transactions['time']))) - value = float(tx['prev_out']['value'] / 100000000) - u, e = convert(value, transactions['time']) - mprint("#" + str(n_tx - i) + "\t" + str(datetime) + "\t-{0:10.8f} BTC {1:10.2f} USD\t{2:10.2f} EUR".format(value, u, e).rstrip('0')) - if script_old != tx['script']: + datetime = time.strftime( + "%d %b %Y %H:%M:%S %Z", + time.localtime(int(transactions["time"])), + ) + value = float(tx["prev_out"]["value"] / 100000000) + u, e = convert(value, transactions["time"]) + mprint( + "#" + + str(n_tx - i) + + "\t" + + str(datetime) + + "\t-{0:10.8f} BTC {1:10.2f} USD\t{2:10.2f} EUR".format(value, u, e).rstrip("0") + ) + if script_old != tx["script"]: i += 1 else: sum_counter += 1 sum += value if sum_counter > 1: - u, e = convert(sum, transactions['time']) + u, e = convert(sum, transactions["time"]) mprint("\t\t\t\t\t----------------------------------------------") - mprint("#" + str(n_tx - i) + "\t\t\t\t Sum:\t-{0:10.8f} BTC {1:10.2f} USD\t{2:10.2f} EUR\n".format(sum, u, e).rstrip('0')) - for tx in transactions['out']: + mprint( + "#" + + str(n_tx - i) + + "\t\t\t\t Sum:\t-{0:10.8f} BTC {1:10.2f} USD\t{2:10.2f} EUR\n".format(sum, u, e).rstrip("0") + ) + for tx in transactions["out"]: try: - addr_out = tx['addr'] + addr_out = tx["addr"] except KeyError: addr_out = None try: - prev_out = tx['prev_out']['value'] + prev_out = tx["prev_out"]["value"] except KeyError: prev_out = None if prev_out != 0 and addr_out == btc: - datetime = time.strftime("%d %b %Y %H:%M:%S %Z", time.localtime(int(transactions['time']))) - value = float(tx['value'] / 100000000) - u, e = convert(value, transactions['time']) - mprint("#" + str(n_tx - i) + "\t" + str(datetime) + "\t {0:10.8f} BTC {1:10.2f} USD\t{2:10.2f} EUR".format(value, u, e).rstrip('0')) + datetime = time.strftime( + "%d %b %Y %H:%M:%S %Z", + time.localtime(int(transactions["time"])), + ) + value = float(tx["value"] / 100000000) + u, e = convert(value, transactions["time"]) + mprint( + "#" + + str(n_tx - i) + + "\t" + + str(datetime) + + "\t {0:10.8f} BTC {1:10.2f} USD\t{2:10.2f} EUR".format(value, u, e).rstrip("0") + ) # i += 1 i += 1 - r = { - 'results': [ - { - 'types': ['text'], - 'values':[ - str(result_text) - ] - } - ] - } + r = {"results": [{"types": ["text"], "values": [str(result_text)]}]} # Debug output on the console print(result_text) # Unset the result for the next request @@ -234,5 +239,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/censys_enrich.py b/misp_modules/modules/expansion/censys_enrich.py index 8531a6069..9e8c41dec 100644 --- a/misp_modules/modules/expansion/censys_enrich.py +++ b/misp_modules/modules/expansion/censys_enrich.py @@ -1,83 +1,98 @@ # encoding: utf-8 -import json -import configparser import base64 import codecs +import configparser +import json + import censys.common.config from dateutil.parser import isoparse -from . import check_input_attribute, standard_error_message from pymisp import MISPAttribute, MISPEvent, MISPObject -try: - #needed in order to overwrite the censys module intent of creating config files in the home folder of the proccess owner - #-- - def get_config_over() -> configparser.ConfigParser: - config = configparser.ConfigParser() - config[censys.common.config.DEFAULT] = censys.common.config.default_config - return config - censys.common.config.get_config = get_config_over - #-- - - from censys.search import CensysHosts - from censys.search import CensysCertificates - from censys.common.base import * -except ImportError: - print("Censys module not installed. Try 'pip install censys'") - -misperrors = {'error': 'Error'} -moduleconfig = ['api_id', 'api_secret'] -mispattributes = {'input': ['ip-src', 'ip-dst', 'domain', 'hostname', 'hostname|port', 'domain|ip', 'ip-dst|port', 'ip-src|port', - 'x509-fingerprint-md5', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256'], 'format': 'misp_standard'} +from . import check_input_attribute, standard_error_message + + +def get_config_over() -> configparser.ConfigParser: + config = configparser.ConfigParser() + config[censys.common.config.DEFAULT] = censys.common.config.default_config + return config + + +censys.common.config.get_config = get_config_over +from censys.common.base import CensysException +from censys.search import CensysCertificates, CensysHosts + +misperrors = {"error": "Error"} +moduleconfig = ["api_id", "api_secret"] +mispattributes = { + "input": [ + "ip-src", + "ip-dst", + "domain", + "hostname", + "hostname|port", + "domain|ip", + "ip-dst|port", + "ip-src|port", + "x509-fingerprint-md5", + "x509-fingerprint-sha1", + "x509-fingerprint-sha256", + ], + "format": "misp_standard", +} moduleinfo = { - 'version': '0.1', - 'author': 'Loïc Fortemps', - 'description': 'An expansion module to enrich attributes in MISP by quering the censys.io API', - 'module-type': ['expansion', 'hover'], - 'name': 'Censys Enrich', - 'logo': '', - 'requirements': ['API credentials to censys.io'], - 'features': 'This module takes an IP, hostname or a certificate fingerprint and attempts to enrich it by querying the Censys API.', - 'references': ['https://www.censys.io'], - 'input': 'IP, domain or certificate fingerprint (md5, sha1 or sha256)', - 'output': 'MISP objects retrieved from censys, including open ports, ASN, Location of the IP, x509 details', + "version": "0.1", + "author": "Loïc Fortemps", + "description": "An expansion module to enrich attributes in MISP by quering the censys.io API", + "module-type": ["expansion", "hover"], + "name": "Censys Enrich", + "logo": "", + "requirements": ["API credentials to censys.io"], + "features": ( + "This module takes an IP, hostname or a certificate fingerprint and attempts to enrich it by querying the" + " Censys API." + ), + "references": ["https://www.censys.io"], + "input": "IP, domain or certificate fingerprint (md5, sha1 or sha256)", + "output": "MISP objects retrieved from censys, including open ports, ASN, Location of the IP, x509 details", } api_id = None api_secret = None + def handler(q=False): global api_id, api_secret if q is False: return False request = json.loads(q) - if request.get('config'): - if (request['config'].get('api_id') is None) or (request['config'].get('api_secret') is None): - misperrors['error'] = "Censys API credentials are missing" + if request.get("config"): + if (request["config"].get("api_id") is None) or (request["config"].get("api_secret") is None): + misperrors["error"] = "Censys API credentials are missing" return misperrors else: - misperrors['error'] = "Please provide config options" + misperrors["error"] = "Please provide config options" return misperrors - api_id = request['config']['api_id'] - api_secret = request['config']['api_secret'] + api_id = request["config"]["api_id"] + api_secret = request["config"]["api_secret"] - if not request.get('attribute') or not check_input_attribute(request['attribute']): - return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} - attribute = request['attribute'] - if not any(input_type == attribute['type'] for input_type in mispattributes['input']): - return {'error': 'Unsupported attribute type.'} + if not request.get("attribute") or not check_input_attribute(request["attribute"]): + return {"error": f"{standard_error_message}, which should contain at least a type, a value and an uuid."} + attribute = request["attribute"] + if not any(input_type == attribute["type"] for input_type in mispattributes["input"]): + return {"error": "Unsupported attribute type."} attribute = MISPAttribute() - attribute.from_dict(**request['attribute']) + attribute.from_dict(**request["attribute"]) # Lists to accomodate multi-types attribute types = list() values = list() results = list() if "|" in attribute.type: - t_1, t_2 = attribute.type.split('|') - v_1, v_2 = attribute.value.split('|') + t_1, t_2 = attribute.type.split("|") + v_1, v_2 = attribute.value.split("|") # We cannot use the port information if t_2 == "port": types.append(t_1) @@ -98,59 +113,59 @@ def handler(q=False): r = CensysHosts(api_id, api_secret).view(value) results.append(parse_response(r, attribute)) found = True - elif t == 'domain' or t == "hostname": + elif t == "domain" or t == "hostname": # get ips endpoint = CensysHosts(api_id, api_secret) for r_list in endpoint.search(query=value, per_page=5, pages=1): for r in r_list: results.append(parse_response(r, attribute)) found = True - elif 'x509-fingerprint-sha256' in t: + elif "x509-fingerprint-sha256" in t: # use api_v1 as Certificates endpoint in api_v2 doesn't yet provide all the details r = CensysCertificates(api_id, api_secret).view(value) results.append(parse_response(r, attribute)) found = True except CensysException as e: - misperrors['error'] = "ERROR: param {} / response: {}".format(value, e) + misperrors["error"] = "ERROR: param {} / response: {}".format(value, e) return misperrors if not found: - misperrors['error'] = "Nothing could be found on Censys" + misperrors["error"] = "Nothing could be found on Censys" return misperrors - return {'results': remove_duplicates(results)} + return {"results": remove_duplicates(results)} def parse_response(censys_output, attribute): misp_event = MISPEvent() misp_event.add_attribute(**attribute) # Generic fields (for IP/Websites) - if censys_output.get('autonomous_system'): - cen_as = censys_output.get('autonomous_system') - asn_object = MISPObject('asn') - asn_object.add_attribute('asn', value=cen_as.get("asn")) - asn_object.add_attribute('description', value=cen_as.get('name')) - asn_object.add_attribute('subnet-announced', value=cen_as.get('routed_prefix')) - asn_object.add_attribute('country', value=cen_as.get('country_code')) - asn_object.add_reference(attribute.uuid, 'associated-to') + if censys_output.get("autonomous_system"): + cen_as = censys_output.get("autonomous_system") + asn_object = MISPObject("asn") + asn_object.add_attribute("asn", value=cen_as.get("asn")) + asn_object.add_attribute("description", value=cen_as.get("name")) + asn_object.add_attribute("subnet-announced", value=cen_as.get("routed_prefix")) + asn_object.add_attribute("country", value=cen_as.get("country_code")) + asn_object.add_reference(attribute.uuid, "associated-to") misp_event.add_object(**asn_object) - if censys_output.get('ip') and len(censys_output.get('services')): #"ports" in censys_output - ip_object = MISPObject('ip-port') - ip_object.add_attribute('ip', value=censys_output.get('ip')) - for serv in censys_output.get('services'): - if serv.get('port'): - ip_object.add_attribute('dst-port', value=serv.get('port')) - ip_object.add_reference(attribute.uuid, 'associated-to') + if censys_output.get("ip") and len(censys_output.get("services")): # "ports" in censys_output + ip_object = MISPObject("ip-port") + ip_object.add_attribute("ip", value=censys_output.get("ip")) + for serv in censys_output.get("services"): + if serv.get("port"): + ip_object.add_attribute("dst-port", value=serv.get("port")) + ip_object.add_reference(attribute.uuid, "associated-to") misp_event.add_object(**ip_object) # We explore all ports to find https or ssh services - for serv in censys_output.get('services', []): + for serv in censys_output.get("services", []): if not isinstance(serv, dict): continue - if serv.get('service_name').lower() == 'http' and serv.get('certificate', None): + if serv.get("service_name").lower() == "http" and serv.get("certificate", None): try: - cert = serv.get('certificate', None) + cert = serv.get("certificate", None) if cert: # TODO switch to api_v2 once available # use api_v1 as Certificates endpoint in api_v2 doesn't yet provide all the details @@ -159,9 +174,9 @@ def parse_response(censys_output, attribute): misp_event.add_object(**cert_obj) except KeyError: print("Error !") - if serv.get('ssh') and serv.get('service_name').lower() == 'ssh': + if serv.get("ssh") and serv.get("service_name").lower() == "ssh": try: - cert = serv.get('ssh').get('server_host_key').get('fingerprint_sha256') + cert = serv.get("ssh").get("server_host_key").get("fingerprint_sha256") # TODO enable once the type is merged # misp_event.add_attribute(type='hasshserver-sha256', value=cert['fingerprint_sha256']) except KeyError: @@ -174,22 +189,22 @@ def parse_response(censys_output, attribute): # Location can be present for IP/Websites results if "location" in censys_output: - loc_obj = MISPObject('geolocation') - loc = censys_output['location'] - loc_obj.add_attribute('latitude', value=loc.get('coordinates', {}).get('latitude', None)) - loc_obj.add_attribute('longitude', value=loc.get('coordinates', {}).get('longitude', None)) - if 'city' in loc: - loc_obj.add_attribute('city', value=loc.get('city')) - loc_obj.add_attribute('country', value=loc.get('country')) - if 'postal_code' in loc: - loc_obj.add_attribute('zipcode', value=loc.get('postal_code')) - if 'province' in loc: - loc_obj.add_attribute('region', value=loc.get('province')) - loc_obj.add_reference(attribute.uuid, 'associated-to') + loc_obj = MISPObject("geolocation") + loc = censys_output["location"] + loc_obj.add_attribute("latitude", value=loc.get("coordinates", {}).get("latitude", None)) + loc_obj.add_attribute("longitude", value=loc.get("coordinates", {}).get("longitude", None)) + if "city" in loc: + loc_obj.add_attribute("city", value=loc.get("city")) + loc_obj.add_attribute("country", value=loc.get("country")) + if "postal_code" in loc: + loc_obj.add_attribute("zipcode", value=loc.get("postal_code")) + if "province" in loc: + loc_obj.add_attribute("region", value=loc.get("province")) + loc_obj.add_reference(attribute.uuid, "associated-to") misp_event.add_object(**loc_obj) event = json.loads(misp_event.to_json()) - return {'Object': event.get('Object', []), 'Attribute': event.get('Attribute', [])} + return {"Object": event.get("Object", []), "Attribute": event.get("Attribute", [])} # In case of multiple enrichment (ip and domain), we need to filter out similar objects @@ -200,19 +215,19 @@ def remove_duplicates(results): return results[0] else: final_result = results[0] - for i,result in enumerate(results[1:]): - obj_l = results[i+1].get('Object', []) + for i, result in enumerate(results[1:]): + obj_l = results[i + 1].get("Object", []) for o2 in obj_l: - if o2['name'] == "asn": + if o2["name"] == "asn": key = "asn" - elif o2['name'] == "ip-port": + elif o2["name"] == "ip-port": key = "ip" - elif o2['name'] == "x509": + elif o2["name"] == "x509": key = "x509-fingerprint-sha256" - elif o2['name'] == "geolocation": + elif o2["name"] == "geolocation": key = "latitude" - if not check_if_present(o2, key, final_result.get('Object', [])): - final_result['Object'].append(o2) + if not check_if_present(o2, key, final_result.get("Object", [])): + final_result["Object"].append(o2) return final_result @@ -226,55 +241,58 @@ def check_if_present(object, attribute_name, list_objects): """ for o in list_objects: # We first look for a match on the name - if o['name'] == object['name']: - for attr in object['Attribute']: + if o["name"] == object["name"]: + for attr in object["Attribute"]: # Within the attributes, we look for the one to compare - if attr['type'] == attribute_name: + if attr["type"] == attribute_name: # Then we check the attributes of the other object and look for a match - for attr2 in o['Attribute']: - if attr2['type'] == attribute_name and attr2['value'] == attr['value']: + for attr2 in o["Attribute"]: + if attr2["type"] == attribute_name and attr2["value"] == attr["value"]: return True return False def get_certificate_object(cert, attribute): - parsed = cert['parsed'] - cert_object = MISPObject('x509') - cert_object.add_attribute('x509-fingerprint-sha256', value=parsed['fingerprint_sha256']) - cert_object.add_attribute('x509-fingerprint-sha1', value=parsed['fingerprint_sha1']) - cert_object.add_attribute('x509-fingerprint-md5', value=parsed['fingerprint_md5']) - cert_object.add_attribute('serial-number', value=parsed['serial_number']) - cert_object.add_attribute('version', value=parsed['version']) - cert_object.add_attribute('subject', value=parsed['subject_dn']) - cert_object.add_attribute('issuer', value=parsed['issuer_dn']) - cert_object.add_attribute('validity-not-before', value=isoparse(parsed['validity']['start'])) - cert_object.add_attribute('validity-not-after', value=isoparse(parsed['validity']['end'])) - cert_object.add_attribute('self_signed', value=parsed['signature']['self_signed']) - cert_object.add_attribute('signature_algorithm', value=parsed['signature']['signature_algorithm']['name']) - - cert_object.add_attribute('pubkey-info-algorithm', value=parsed['subject_key_info']['key_algorithm']['name']) - - if 'rsa_public_key' in parsed['subject_key_info']: - pub_key = parsed['subject_key_info']['rsa_public_key'] - cert_object.add_attribute('pubkey-info-size', value=pub_key['length']) - cert_object.add_attribute('pubkey-info-exponent', value=pub_key['exponent']) - hex_mod = codecs.encode(base64.b64decode(pub_key['modulus']), 'hex').decode() - cert_object.add_attribute('pubkey-info-modulus', value=hex_mod) + parsed = cert["parsed"] + cert_object = MISPObject("x509") + cert_object.add_attribute("x509-fingerprint-sha256", value=parsed["fingerprint_sha256"]) + cert_object.add_attribute("x509-fingerprint-sha1", value=parsed["fingerprint_sha1"]) + cert_object.add_attribute("x509-fingerprint-md5", value=parsed["fingerprint_md5"]) + cert_object.add_attribute("serial-number", value=parsed["serial_number"]) + cert_object.add_attribute("version", value=parsed["version"]) + cert_object.add_attribute("subject", value=parsed["subject_dn"]) + cert_object.add_attribute("issuer", value=parsed["issuer_dn"]) + cert_object.add_attribute("validity-not-before", value=isoparse(parsed["validity"]["start"])) + cert_object.add_attribute("validity-not-after", value=isoparse(parsed["validity"]["end"])) + cert_object.add_attribute("self_signed", value=parsed["signature"]["self_signed"]) + cert_object.add_attribute("signature_algorithm", value=parsed["signature"]["signature_algorithm"]["name"]) + + cert_object.add_attribute( + "pubkey-info-algorithm", + value=parsed["subject_key_info"]["key_algorithm"]["name"], + ) + + if "rsa_public_key" in parsed["subject_key_info"]: + pub_key = parsed["subject_key_info"]["rsa_public_key"] + cert_object.add_attribute("pubkey-info-size", value=pub_key["length"]) + cert_object.add_attribute("pubkey-info-exponent", value=pub_key["exponent"]) + hex_mod = codecs.encode(base64.b64decode(pub_key["modulus"]), "hex").decode() + cert_object.add_attribute("pubkey-info-modulus", value=hex_mod) if "extensions" in parsed and "subject_alt_name" in parsed["extensions"]: san = parsed["extensions"]["subject_alt_name"] if "dns_names" in san: - for dns in san['dns_names']: - cert_object.add_attribute('dns_names', value=dns) + for dns in san["dns_names"]: + cert_object.add_attribute("dns_names", value=dns) if "ip_addresses" in san: - for ip in san['ip_addresses']: - cert_object.add_attribute('ip', value=ip) + for ip in san["ip_addresses"]: + cert_object.add_attribute("ip", value=ip) if "raw" in cert: - cert_object.add_attribute('raw-base64', value=cert['raw']) + cert_object.add_attribute("raw-base64", value=cert["raw"]) - cert_object.add_reference(attribute.uuid, 'associated-to') + cert_object.add_reference(attribute.uuid, "associated-to") return cert_object @@ -283,5 +301,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig - return moduleinfo \ No newline at end of file + moduleinfo["config"] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/circl_passivedns.py b/misp_modules/modules/expansion/circl_passivedns.py index dc2084ad9..03e524ded 100755 --- a/misp_modules/modules/expansion/circl_passivedns.py +++ b/misp_modules/modules/expansion/circl_passivedns.py @@ -1,23 +1,37 @@ import pypdns -from . import check_input_attribute, standard_error_message from pymisp import MISPAttribute, MISPEvent, MISPObject -mispattributes = {'input': ['hostname', 'domain', 'ip-src', 'ip-dst', 'ip-src|port', 'ip-dst|port'], 'format': 'misp_standard'} +from . import check_input_attribute, standard_error_message + +mispattributes = { + "input": ["hostname", "domain", "ip-src", "ip-dst", "ip-src|port", "ip-dst|port"], + "format": "misp_standard", +} moduleinfo = { - 'version': '0.2', - 'author': 'Alexandre Dulaunoy', - 'description': 'Module to access CIRCL Passive DNS.', - 'module-type': ['expansion', 'hover'], - 'name': 'CIRCL Passive DNS', - 'logo': 'passivedns.png', - 'requirements': ['pypdns: Passive DNS python library', 'A CIRCL passive DNS account with username & password'], - 'features': 'This module takes a hostname, domain or ip-address (ip-src or ip-dst) attribute as input, and queries the CIRCL Passive DNS REST API to get the asssociated passive dns entries and return them as MISP objects.\n\nTo make it work a username and a password are thus required to authenticate to the CIRCL Passive DNS API.', - 'references': ['https://www.circl.lu/services/passive-dns/', 'https://datatracker.ietf.org/doc/draft-dulaunoy-dnsop-passive-dns-cof/'], - 'input': 'Hostname, domain, or ip-address attribute.', - 'output': '', - 'ouput': 'Passive DNS objects related to the input attribute.', + "version": "0.2", + "author": "Alexandre Dulaunoy", + "description": "Module to access CIRCL Passive DNS.", + "module-type": ["expansion", "hover"], + "name": "CIRCL Passive DNS", + "logo": "passivedns.png", + "requirements": [ + "pypdns: Passive DNS python library", + "A CIRCL passive DNS account with username & password", + ], + "features": ( + "This module takes a hostname, domain or ip-address (ip-src or ip-dst) attribute as input, and queries the" + " CIRCL Passive DNS REST API to get the asssociated passive dns entries and return them as MISP objects.\n\nTo" + " make it work a username and a password are thus required to authenticate to the CIRCL Passive DNS API." + ), + "references": [ + "https://www.circl.lu/services/passive-dns/", + "https://datatracker.ietf.org/doc/draft-dulaunoy-dnsop-passive-dns-cof/", + ], + "input": "Hostname, domain, or ip-address attribute.", + "output": "", + "ouput": "Passive DNS objects related to the input attribute.", } -moduleconfig = ['username', 'password'] +moduleconfig = ["username", "password"] class PassiveDNSParser: @@ -29,54 +43,59 @@ def __init__(self, attribute, authentication): self.pdns = pypdns.PyPDNS(basic_auth=authentication) def get_results(self): - if hasattr(self, 'result'): + if hasattr(self, "result"): return self.result event = self.misp_event.to_dict() - results = {key: event[key] for key in ('Attribute', 'Object')} - return {'results': results} + results = {key: event[key] for key in ("Attribute", "Object")} + return {"results": results} def parse(self): - value = self.attribute.value.split('|')[0] if '|' in self.attribute.type else self.attribute.value + value = self.attribute.value.split("|")[0] if "|" in self.attribute.type else self.attribute.value try: results = self.pdns.query(value) except Exception: - self.result = {'error': 'There is an authentication error, please make sure you supply correct credentials.'} + self.result = { + "error": "There is an authentication error, please make sure you supply correct credentials." + } return if not results: - self.result = {'error': 'Not found'} + self.result = {"error": "Not found"} return mapping = { - 'count': 'counter', 'origin': 'text', 'rrtype': 'text', - 'rrname': 'text', 'rdata': 'text', + "count": "counter", + "origin": "text", + "rrtype": "text", + "rrname": "text", + "rdata": "text", } for result in results: - pdns_object = MISPObject('passive-dns') + pdns_object = MISPObject("passive-dns") for relation, attribute_type in mapping.items(): pdns_object.add_attribute(relation, result[relation], type=attribute_type) - first_seen = result['time_first'] - pdns_object.add_attribute('time_first', first_seen, type='datetime') + first_seen = result["time_first"] + pdns_object.add_attribute("time_first", first_seen, type="datetime") pdns_object.first_seen = first_seen - last_seen = result['time_last'] - pdns_object.add_attribute('time_last', last_seen, type='datetime') + last_seen = result["time_last"] + pdns_object.add_attribute("time_last", last_seen, type="datetime") pdns_object.last_seen = last_seen - pdns_object.add_reference(self.attribute.uuid, 'associated-to') + pdns_object.add_reference(self.attribute.uuid, "associated-to") self.misp_event.add_object(**pdns_object) def dict_handler(request: dict): - if not request.get('config'): - return {'error': 'CIRCL Passive DNS authentication is missing.'} - if not request['config'].get('username') or not request['config'].get('password'): - return {'error': 'CIRCL Passive DNS authentication is incomplete, please provide your username and password.'} - authentication = (request['config']['username'], request['config']['password']) - if not request.get('attribute') or not check_input_attribute(request['attribute']): - return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} - attribute = request['attribute'] - if not any(input_type == attribute['type'] for input_type in mispattributes['input']): - return {'error': 'Unsupported attribute type.'} + if not request.get("config"): + return {"error": "CIRCL Passive DNS authentication is missing."} + if not request["config"].get("username") or not request["config"].get("password"): + return {"error": "CIRCL Passive DNS authentication is incomplete, please provide your username and password."} + authentication = (request["config"]["username"], request["config"]["password"]) + if not request.get("attribute") or not check_input_attribute(request["attribute"]): + return {"error": f"{standard_error_message}, which should contain at least a type, a value and an uuid."} + attribute = request["attribute"] + if not any(input_type == attribute["type"] for input_type in mispattributes["input"]): + return {"error": "Unsupported attribute type."} pdns_parser = PassiveDNSParser(attribute, authentication) pdns_parser.parse() return pdns_parser.get_results() @@ -87,5 +106,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/circl_passivessl.py b/misp_modules/modules/expansion/circl_passivessl.py index e04adcf12..3ceb6b996 100755 --- a/misp_modules/modules/expansion/circl_passivessl.py +++ b/misp_modules/modules/expansion/circl_passivessl.py @@ -1,86 +1,102 @@ import json + import pypssl -from . import check_input_attribute, standard_error_message from pymisp import MISPAttribute, MISPEvent, MISPObject -mispattributes = {'input': ['ip-src', 'ip-dst', 'ip-src|port', 'ip-dst|port'], 'format': 'misp_standard'} +from . import check_input_attribute, standard_error_message + +mispattributes = { + "input": ["ip-src", "ip-dst", "ip-src|port", "ip-dst|port"], + "format": "misp_standard", +} moduleinfo = { - 'version': '0.2', - 'author': 'Raphaël Vinot', - 'description': 'Modules to access CIRCL Passive SSL.', - 'module-type': ['expansion', 'hover'], - 'name': 'CIRCL Passive SSL', - 'logo': 'passivessl.png', - 'requirements': ['pypssl: Passive SSL python library', 'A CIRCL passive SSL account with username & password'], - 'features': 'This module takes an ip-address (ip-src or ip-dst) attribute as input, and queries the CIRCL Passive SSL REST API to gather the related certificates and return the corresponding MISP objects.\n\nTo make it work a username and a password are required to authenticate to the CIRCL Passive SSL API.', - 'references': ['https://www.circl.lu/services/passive-ssl/'], - 'input': 'IP address attribute.', - 'output': 'x509 certificate objects seen by the IP address(es).', + "version": "0.2", + "author": "Raphaël Vinot", + "description": "Modules to access CIRCL Passive SSL.", + "module-type": ["expansion", "hover"], + "name": "CIRCL Passive SSL", + "logo": "passivessl.png", + "requirements": [ + "pypssl: Passive SSL python library", + "A CIRCL passive SSL account with username & password", + ], + "features": ( + "This module takes an ip-address (ip-src or ip-dst) attribute as input, and queries the CIRCL Passive SSL REST" + " API to gather the related certificates and return the corresponding MISP objects.\n\nTo make it work a" + " username and a password are required to authenticate to the CIRCL Passive SSL API." + ), + "references": ["https://www.circl.lu/services/passive-ssl/"], + "input": "IP address attribute.", + "output": "x509 certificate objects seen by the IP address(es).", } -moduleconfig = ['username', 'password'] +moduleconfig = ["username", "password"] -class PassiveSSLParser(): +class PassiveSSLParser: def __init__(self, attribute, authentication): self.misp_event = MISPEvent() self.attribute = MISPAttribute() self.attribute.from_dict(**attribute) self.misp_event.add_attribute(**self.attribute) self.pssl = pypssl.PyPSSL(basic_auth=authentication) - self.cert_hash = 'x509-fingerprint-sha1' - self.cert_type = 'pem' - self.mapping = {'issuer': ('text', 'issuer'), - 'keylength': ('text', 'pubkey-info-size'), - 'not_after': ('datetime', 'validity-not-after'), - 'not_before': ('datetime', 'validity-not-before'), - 'subject': ('text', 'subject')} + self.cert_hash = "x509-fingerprint-sha1" + self.cert_type = "pem" + self.mapping = { + "issuer": ("text", "issuer"), + "keylength": ("text", "pubkey-info-size"), + "not_after": ("datetime", "validity-not-after"), + "not_before": ("datetime", "validity-not-before"), + "subject": ("text", "subject"), + } def get_results(self): - if hasattr(self, 'result'): + if hasattr(self, "result"): return self.result event = json.loads(self.misp_event.to_json()) - results = {key: event[key] for key in ('Attribute', 'Object')} - return {'results': results} + results = {key: event[key] for key in ("Attribute", "Object")} + return {"results": results} def parse(self): - value = self.attribute.value.split('|')[0] if '|' in self.attribute.type else self.attribute.value + value = self.attribute.value.split("|")[0] if "|" in self.attribute.type else self.attribute.value try: results = self.pssl.query(value) except Exception: - self.result = {'error': 'There is an authentication error, please make sure you supply correct credentials.'} + self.result = { + "error": "There is an authentication error, please make sure you supply correct credentials." + } return if not results: - self.result = {'error': 'Not found'} + self.result = {"error": "Not found"} return - if 'error' in results: - self.result = {'error': results['error']} + if "error" in results: + self.result = {"error": results["error"]} return for ip_address, certificates in results.items(): ip_uuid = self._handle_ip_attribute(ip_address) - for certificate in certificates['certificates']: + for certificate in certificates["certificates"]: self._handle_certificate(certificate, ip_uuid) def _handle_certificate(self, certificate, ip_uuid): - x509 = MISPObject('x509') + x509 = MISPObject("x509") x509.add_attribute(self.cert_hash, type=self.cert_hash, value=certificate) cert_details = self.pssl.fetch_cert(certificate) - info = cert_details['info'] + info = cert_details["info"] for feature, mapping in self.mapping.items(): attribute_type, object_relation = mapping x509.add_attribute(object_relation, type=attribute_type, value=info[feature]) - x509.add_attribute(self.cert_type, type='text', value=self.cert_type) - x509.add_reference(ip_uuid, 'seen-by') + x509.add_attribute(self.cert_type, type="text", value=self.cert_type) + x509.add_reference(ip_uuid, "seen-by") self.misp_event.add_object(**x509) def _handle_ip_attribute(self, ip_address): if ip_address == self.attribute.value: return self.attribute.uuid ip_attribute = MISPAttribute() - ip_attribute.from_dict(**{'type': self.attribute.type, 'value': ip_address}) + ip_attribute.from_dict(**{"type": self.attribute.type, "value": ip_address}) self.misp_event.add_attribute(**ip_attribute) return ip_attribute.uuid @@ -89,16 +105,16 @@ def handler(q=False): if q is False: return False request = json.loads(q) - if not request.get('config'): - return {'error': 'CIRCL Passive SSL authentication is missing.'} - if not request['config'].get('username') or not request['config'].get('password'): - return {'error': 'CIRCL Passive SSL authentication is incomplete, please provide your username and password.'} - authentication = (request['config']['username'], request['config']['password']) - if not request.get('attribute') or not check_input_attribute(request['attribute']): - return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} - attribute = request['attribute'] - if not any(input_type == attribute['type'] for input_type in mispattributes['input']): - return {'error': 'Unsupported attribute type.'} + if not request.get("config"): + return {"error": "CIRCL Passive SSL authentication is missing."} + if not request["config"].get("username") or not request["config"].get("password"): + return {"error": "CIRCL Passive SSL authentication is incomplete, please provide your username and password."} + authentication = (request["config"]["username"], request["config"]["password"]) + if not request.get("attribute") or not check_input_attribute(request["attribute"]): + return {"error": f"{standard_error_message}, which should contain at least a type, a value and an uuid."} + attribute = request["attribute"] + if not any(input_type == attribute["type"] for input_type in mispattributes["input"]): + return {"error": "Unsupported attribute type."} pssl_parser = PassiveSSLParser(attribute, authentication) pssl_parser.parse() return pssl_parser.get_results() @@ -109,5 +125,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/clamav.py b/misp_modules/modules/expansion/clamav.py index 61d848dfa..e98a1dca0 100644 --- a/misp_modules/modules/expansion/clamav.py +++ b/misp_modules/modules/expansion/clamav.py @@ -3,39 +3,36 @@ import logging import sys import zipfile -import clamd -from . import check_input_attribute, standard_error_message from typing import Optional + +import clamd from pymisp import MISPEvent, MISPObject +from . import check_input_attribute, standard_error_message + log = logging.getLogger("clamav") log.setLevel(logging.DEBUG) sh = logging.StreamHandler(sys.stdout) sh.setLevel(logging.DEBUG) -fmt = logging.Formatter( - "%(asctime)s - %(name)s - %(levelname)s - %(message)s" -) +fmt = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") sh.setFormatter(fmt) log.addHandler(sh) moduleinfo = { - 'version': '0.1', - 'author': 'Jakub Onderka', - 'description': 'Submit file to ClamAV', - 'module-type': ['expansion'], - 'name': 'ClaamAV', - 'logo': '', - 'requirements': [], - 'features': '', - 'references': [], - 'input': '', - 'output': '', + "version": "0.1", + "author": "Jakub Onderka", + "description": "Submit file to ClamAV", + "module-type": ["expansion"], + "name": "ClaamAV", + "logo": "", + "requirements": [], + "features": "", + "references": [], + "input": "", + "output": "", } moduleconfig = ["connection"] -mispattributes = { - "input": ["attachment", "malware-sample"], - "format": "misp_standard" -} +mispattributes = {"input": ["attachment", "malware-sample"], "format": "misp_standard"} def create_response(original_attribute: dict, software: str, signature: Optional[str] = None) -> dict: @@ -50,7 +47,7 @@ def create_response(original_attribute: dict, software: str, signature: Optional misp_event.add_object(av_signature_object) event = misp_event.to_dict() - results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} + results = {key: event[key] for key in ("Attribute", "Object") if (key in event and event[key])} return {"results": results} @@ -61,7 +58,9 @@ def connect_to_clamav(connection_string: str) -> clamd.ClamdNetworkSocket: host, port = connection_string.split(":") return clamd.ClamdNetworkSocket(host, int(port)) else: - raise Exception("ClamAV connection string is invalid. It must be unix socket path with 'unix://' prefix or IP:PORT.") + raise Exception( + "ClamAV connection string is invalid. It must be unix socket path with 'unix://' prefix or IP:PORT." + ) def dict_handler(request: dict): @@ -73,8 +72,8 @@ def dict_handler(request: dict): if not attribute: return {"error": "No attribute provided"} - if not check_input_attribute(request['attribute']): - return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} + if not check_input_attribute(request["attribute"]): + return {"error": f"{standard_error_message}, which should contain at least a type, a value and an uuid."} if attribute["type"] not in mispattributes["input"]: return {"error": "Invalid attribute type provided, expected 'malware-sample' or 'attachment'"} diff --git a/misp_modules/modules/expansion/cluster25_expand.py b/misp_modules/modules/expansion/cluster25_expand.py index 31eb63616..4324e2e2a 100755 --- a/misp_modules/modules/expansion/cluster25_expand.py +++ b/misp_modules/modules/expansion/cluster25_expand.py @@ -1,45 +1,67 @@ import json -import requests import uuid -from . import check_input_attribute, standard_error_message + +import requests from pymisp import MISPAttribute, MISPEvent, MISPObject +from . import check_input_attribute, standard_error_message + moduleinfo = { - 'version': '0.1', - 'author': 'Milo Volpicelli', - 'description': 'Module to query Cluster25 CTI.', - 'module-type': ['expansion', 'hover'], - 'name': 'Cluster25 Expand', - 'logo': 'cluster25.png', - 'requirements': ['A Cluster25 API access (API id & key)'], - 'features': 'This module takes a MISP attribute value as input to query the Cluster25CTI API. The result is then mapped into compatible MISP Objects and relative attributes.\n', - 'references': [''], - 'input': 'An Indicator value of type included in the following list:\n- domain\n- email-src\n- email-dst\n- filename\n- md5\n- sha1\n- sha256\n- ip-src\n- ip-dst\n- url\n- vulnerability\n- btc\n- xmr\n ja3-fingerprint-md5', - 'output': 'A series of c25 MISP Objects with colletion of attributes mapped from Cluster25 CTI query result.', + "version": "0.1", + "author": "Milo Volpicelli", + "description": "Module to query Cluster25 CTI.", + "module-type": ["expansion", "hover"], + "name": "Cluster25 Expand", + "logo": "cluster25.png", + "requirements": ["A Cluster25 API access (API id & key)"], + "features": ( + "This module takes a MISP attribute value as input to query the Cluster25CTI API. The result is then mapped" + " into compatible MISP Objects and relative attributes.\n" + ), + "references": [""], + "input": ( + "An Indicator value of type included in the following list:\n- domain\n- email-src\n- email-dst\n- filename\n-" + " md5\n- sha1\n- sha256\n- ip-src\n- ip-dst\n- url\n- vulnerability\n- btc\n- xmr\n ja3-fingerprint-md5" + ), + "output": "A series of c25 MISP Objects with colletion of attributes mapped from Cluster25 CTI query result.", } -moduleconfig = ['api_id', 'apikey', 'base_url'] -misperrors = {'error': 'Error'} -misp_type_in = ['domain', 'email-src', 'email-dst', 'filename', 'md5', 'sha1', 'sha256', 'ip-src', 'ip-dst', 'url', - 'vulnerability', 'btc', 'xmr', 'ja3-fingerprint-md5'] +moduleconfig = ["api_id", "apikey", "base_url"] +misperrors = {"error": "Error"} +misp_type_in = [ + "domain", + "email-src", + "email-dst", + "filename", + "md5", + "sha1", + "sha256", + "ip-src", + "ip-dst", + "url", + "vulnerability", + "btc", + "xmr", + "ja3-fingerprint-md5", +] mapping_out = { # mapping between the MISP attributes type and the compatible Cluster25 indicator types. - 'domain': {'type': 'domain', 'to_ids': True}, - 'email-src': {'type': 'email-src', 'to_ids': True}, - 'email-dst': {'type': 'email-dst', 'to_ids': True}, - 'filename': {'type': 'filename', 'to_ids': True}, - 'md5': {'type': 'md5', 'to_ids': True}, - 'sha1': {'type': 'sha1', 'to_ids': True}, - 'sha256': {'type': 'sha256', 'to_ids': True}, - 'ip-src': {'type': 'ip-src', 'to_ids': True}, - 'ip-dst': {'type': 'ip-dst', 'to_ids': True}, - 'url': {'type': 'url', 'to_ids': True}, - 'cve': {'type': 'vulnerability', 'to_ids': True}, - 'btcaddress': {'type': 'btc', 'to_ids': True}, - 'xmraddress': {'type': 'xmr', 'to_ids': True}, - 'ja3': {'type': 'ja3-fingerprint-md5', 'to_ids': True}, + "domain": {"type": "domain", "to_ids": True}, + "email-src": {"type": "email-src", "to_ids": True}, + "email-dst": {"type": "email-dst", "to_ids": True}, + "filename": {"type": "filename", "to_ids": True}, + "md5": {"type": "md5", "to_ids": True}, + "sha1": {"type": "sha1", "to_ids": True}, + "sha256": {"type": "sha256", "to_ids": True}, + "ip-src": {"type": "ip-src", "to_ids": True}, + "ip-dst": {"type": "ip-dst", "to_ids": True}, + "url": {"type": "url", "to_ids": True}, + "cve": {"type": "vulnerability", "to_ids": True}, + "btcaddress": {"type": "btc", "to_ids": True}, + "xmraddress": {"type": "xmr", "to_ids": True}, + "ja3": {"type": "ja3-fingerprint-md5", "to_ids": True}, } -misp_type_out = [item['type'] for item in mapping_out.values()] -misp_attributes = {'input': misp_type_in, 'format': 'misp_standard'} +misp_type_out = [item["type"] for item in mapping_out.values()] +misp_attributes = {"input": misp_type_in, "format": "misp_standard"} def handler(q=False): @@ -47,27 +69,31 @@ def handler(q=False): return False request = json.loads(q) # validate Cluster25 params - if request.get('config'): - if request['config'].get('apikey') is None: - misperrors['error'] = 'Cluster25 apikey is missing' + if request.get("config"): + if request["config"].get("apikey") is None: + misperrors["error"] = "Cluster25 apikey is missing" return misperrors - if request['config'].get('api_id') is None: - misperrors['error'] = 'Cluster25 api_id is missing' + if request["config"].get("api_id") is None: + misperrors["error"] = "Cluster25 api_id is missing" return misperrors - if request['config'].get('base_url') is None: - misperrors['error'] = 'Cluster25 base_url is missing' + if request["config"].get("base_url") is None: + misperrors["error"] = "Cluster25 base_url is missing" return misperrors # validate attribute - if not request.get('attribute') or not check_input_attribute(request['attribute']): - return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} - attribute = request.get('attribute') - if not any(input_type == attribute.get('type') for input_type in misp_type_in): - return {'error': 'Unsupported attribute type.'} + if not request.get("attribute") or not check_input_attribute(request["attribute"]): + return {"error": f"{standard_error_message}, which should contain at least a type, a value and an uuid."} + attribute = request.get("attribute") + if not any(input_type == attribute.get("type") for input_type in misp_type_in): + return {"error": "Unsupported attribute type."} - client = Cluster25CTI(request['config']['api_id'], request['config']['apikey'], request['config']['base_url']) + client = Cluster25CTI( + request["config"]["api_id"], + request["config"]["apikey"], + request["config"]["base_url"], + ) - return lookup_indicator(client, request.get('attribute')) + return lookup_indicator(client, request.get("attribute")) def format_content(content): @@ -100,17 +126,17 @@ def format_content(content): def lookup_indicator(client, attr): result = client.investigate(attr) - if result.get('error'): + if result.get("error"): return result misp_event = MISPEvent() attribute = MISPAttribute() attribute.from_dict(**attr) misp_event.add_attribute(**attribute) - misp_object_g = MISPObject('c25_generic_info') + misp_object_g = MISPObject("c25_generic_info") misp_object_g.template_uuid = uuid.uuid4() - misp_object_g.description = 'c25_generic_info' - setattr(misp_object_g, 'meta-category', 'network') + misp_object_g.description = "c25_generic_info" + setattr(misp_object_g, "meta-category", "network") misp_objects = [] for ind, entry in enumerate(result): @@ -118,14 +144,19 @@ def lookup_indicator(client, attr): tmp_obj = MISPObject(f"c25_{entry}") tmp_obj.template_uuid = uuid.uuid4() tmp_obj.description = f"c25_{entry}" - setattr(tmp_obj, 'meta-category', 'network') - tmp_obj.add_reference(attribute['uuid'], 'related-to') + setattr(tmp_obj, "meta-category", "network") + tmp_obj.add_reference(attribute["uuid"], "related-to") for key in result[entry]: if isinstance(result[entry][key], dict): for index, item in enumerate(result[entry][key]): if result[entry][key][item]: - tmp_obj.add_attribute(f"{entry}_{key}_{item}", **{'type': 'text', 'value': format_content( - result[entry][key][item])}) + tmp_obj.add_attribute( + f"{entry}_{key}_{item}", + **{ + "type": "text", + "value": format_content(result[entry][key][item]), + }, + ) elif isinstance(result[entry][key], list): for index, item in enumerate(result[entry][key]): @@ -133,8 +164,8 @@ def lookup_indicator(client, attr): tmp_obj_2 = MISPObject(f"c25_{entry}_{key}_{index + 1}") tmp_obj_2.template_uuid = uuid.uuid4() tmp_obj_2.description = f"c25_{entry}_{key}" - setattr(tmp_obj_2, 'meta-category', 'network') - tmp_obj_2.add_reference(attribute['uuid'], 'related-to') + setattr(tmp_obj_2, "meta-category", "network") + tmp_obj_2.add_reference(attribute["uuid"], "related-to") for sub_key in item: if isinstance(item[sub_key], list): for sub_item in item[sub_key]: @@ -142,35 +173,54 @@ def lookup_indicator(client, attr): tmp_obj_3 = MISPObject(f"c25_{entry}_{sub_key}_{index + 1}") tmp_obj_3.template_uuid = uuid.uuid4() tmp_obj_3.description = f"c25_{entry}_{sub_key}" - setattr(tmp_obj_3, 'meta-category', 'network') - tmp_obj_3.add_reference(attribute['uuid'], 'related-to') + setattr(tmp_obj_3, "meta-category", "network") + tmp_obj_3.add_reference(attribute["uuid"], "related-to") for sub_sub_key in sub_item: if isinstance(sub_item[sub_sub_key], list): for idx, sub_sub_item in enumerate(sub_item[sub_sub_key]): if sub_sub_item.get("name"): sub_sub_item = sub_sub_item.get("name") - tmp_obj_3.add_attribute(f"{sub_sub_key}_{idx + 1}", - **{'type': 'text', - 'value': format_content( - sub_sub_item)}) + tmp_obj_3.add_attribute( + f"{sub_sub_key}_{idx + 1}", + **{ + "type": "text", + "value": format_content(sub_sub_item), + }, + ) else: - tmp_obj_3.add_attribute(sub_sub_key, - **{'type': 'text', - 'value': format_content( - sub_item[sub_sub_key])}) + tmp_obj_3.add_attribute( + sub_sub_key, + **{ + "type": "text", + "value": format_content(sub_item[sub_sub_key]), + }, + ) misp_objects.append(tmp_obj_3) else: - tmp_obj_2.add_attribute(sub_key, **{'type': 'text', - 'value': format_content(sub_item)}) + tmp_obj_2.add_attribute( + sub_key, + **{ + "type": "text", + "value": format_content(sub_item), + }, + ) elif item[sub_key]: - tmp_obj_2.add_attribute(sub_key, - **{'type': 'text', 'value': format_content(item[sub_key])}) + tmp_obj_2.add_attribute( + sub_key, + **{ + "type": "text", + "value": format_content(item[sub_key]), + }, + ) misp_objects.append(tmp_obj_2) elif item is not None: - tmp_obj.add_attribute(f"{entry}_{key}", **{'type': 'text', 'value': format_content(item)}) + tmp_obj.add_attribute( + f"{entry}_{key}", + **{"type": "text", "value": format_content(item)}, + ) elif result[entry][key] is not None: - tmp_obj.add_attribute(key, **{'type': 'text', 'value': result[entry][key]}) + tmp_obj.add_attribute(key, **{"type": "text", "value": result[entry][key]}) if tmp_obj.attributes: misp_objects.append(tmp_obj) @@ -181,28 +231,33 @@ def lookup_indicator(client, attr): tmp_obj = MISPObject(f"c25_{entry}_{index + 1}") tmp_obj.template_uuid = uuid.uuid4() tmp_obj.description = f"c25_{entry}_{index + 1}" - setattr(tmp_obj, 'meta-category', 'network') - tmp_obj.add_reference(attribute['uuid'], 'related-to') + setattr(tmp_obj, "meta-category", "network") + tmp_obj.add_reference(attribute["uuid"], "related-to") for item in key: if key[item]: - tmp_obj.add_attribute(item, **{'type': 'text', 'value': format_content(key[item])}) - tmp_obj.add_reference(attribute['uuid'], 'related-to') + tmp_obj.add_attribute( + item, + **{"type": "text", "value": format_content(key[item])}, + ) + tmp_obj.add_reference(attribute["uuid"], "related-to") misp_objects.append(tmp_obj) elif key is not None: - misp_object_g.add_attribute(f"{entry}_{index + 1}", - **{'type': 'text', 'value': format_content(key)}) + misp_object_g.add_attribute( + f"{entry}_{index + 1}", + **{"type": "text", "value": format_content(key)}, + ) else: if result[entry]: - misp_object_g.add_attribute(entry, **{'type': 'text', 'value': result[entry]}) + misp_object_g.add_attribute(entry, **{"type": "text", "value": result[entry]}) - misp_object_g.add_reference(attribute['uuid'], 'related-to') + misp_object_g.add_reference(attribute["uuid"], "related-to") misp_event.add_object(misp_object_g) for misp_object in misp_objects: misp_event.add_object(misp_object) event = json.loads(misp_event.to_json()) - results = {key: event[key] for key in ('Attribute', 'Object')} - return {'results': results} + results = {key: event[key] for key in ("Attribute", "Object")} + return {"results": results} def introspection(): @@ -210,7 +265,7 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo @@ -224,16 +279,23 @@ def __init__(self, customer_id=None, customer_key=None, base_url=None): def _get_cluster25_token(self): payload = {"client_id": self.client_id, "client_secret": self.client_secret} - r = requests.post(url=f"{self.base_url}/token", json=payload, headers={"Content-Type": "application/json"}) + r = requests.post( + url=f"{self.base_url}/token", + json=payload, + headers={"Content-Type": "application/json"}, + ) if r.status_code != 200: - return {'error': f"Unable to retrieve the token from C25 platform, status {r.status_code}"} + return {"error": f"Unable to retrieve the token from C25 platform, status {r.status_code}"} return r.json()["data"]["token"] def investigate(self, indicator) -> dict: - params = {'indicator': indicator.get('value')} + params = {"indicator": indicator.get("value")} r = requests.get(url=f"{self.base_url}/investigate", params=params, headers=self.headers) if r.status_code != 200: - return {'error': f"Unable to retrieve investigate result for indicator '{indicator.get('value')}' " - f"from C25 platform, status {r.status_code}"} + return { + "error": ( + f"Unable to retrieve investigate result for indicator '{indicator.get('value')}' " + f"from C25 platform, status {r.status_code}" + ) + } return r.json()["data"] - diff --git a/misp_modules/modules/expansion/convert_markdown_to_pdf.py b/misp_modules/modules/expansion/convert_markdown_to_pdf.py index 656cee599..5b9ef4dcc 100755 --- a/misp_modules/modules/expansion/convert_markdown_to_pdf.py +++ b/misp_modules/modules/expansion/convert_markdown_to_pdf.py @@ -1,16 +1,16 @@ #!/usr/bin/env python\ -import json import base64 -import pandoc +import json +import os import random +import shutil import string import subprocess -import os -import shutil +import pandoc -installationNotes = ''' +installationNotes = """ 1. Install pandoc for your distribution 2. Install wkhtmltopdf - Ensure You have install the version with patched qt @@ -23,33 +23,37 @@ ```bash pip3 install git+https://github.com/DavidCruciani/pandoc-mermaid-filter ``` -''' +""" -misperrors = {'error': 'Error'} -mispattributes = {'input': ['text'], 'output': ['text']} +misperrors = {"error": "Error"} +mispattributes = {"input": ["text"], "output": ["text"]} moduleinfo = { - 'version': '0.3', - 'author': 'Sami Mokaddem', - 'description': 'Render the markdown (under GFM) into PDF. Requires pandoc (https://pandoc.org/), wkhtmltopdf (https://wkhtmltopdf.org/) and mermaid dependencies.', - 'module-type': ['expansion'], - 'name': 'Markdown to PDF converter', - 'logo': '', - 'requirements': ['pandoc'], - 'features': '', - 'references': [installationNotes], - 'input': '', - 'output': '', + "version": "0.3", + "author": "Sami Mokaddem", + "description": ( + "Render the markdown (under GFM) into PDF. Requires pandoc (https://pandoc.org/), wkhtmltopdf" + " (https://wkhtmltopdf.org/) and mermaid dependencies." + ), + "module-type": ["expansion"], + "name": "Markdown to PDF converter", + "logo": "", + "requirements": ["pandoc"], + "features": "", + "references": [installationNotes], + "input": "", + "output": "", } -moduleconfig = [ -] +moduleconfig = [] + def randomFilename(length=10): characters = string.ascii_lowercase + string.digits # Lowercase letters and digits - return ''.join(random.choices(characters, k=length)) + return "".join(random.choices(characters, k=length)) -def convert(markdown, margin='3'): - doc = pandoc.read(markdown, format='gfm') + +def convert(markdown, margin="3"): + doc = pandoc.read(markdown, format="gfm") elt = doc @@ -95,20 +99,27 @@ def convert(markdown, margin='3'): randomFn = randomFilename() command = [ "/usr/bin/pandoc", - "-t", "pdf", - "-o", f"/tmp/{randomFn}/output", + "-t", + "pdf", + "-o", + f"/tmp/{randomFn}/output", "--pdf-engine=wkhtmltopdf", - "-V", f"margin-left={margin}", - "-V", f"margin-right={margin}", - "-V", f"margin-top={margin}", - "-V", f"margin-bottom={margin}", + "-V", + f"margin-left={margin}", + "-V", + f"margin-right={margin}", + "-V", + f"margin-top={margin}", + "-V", + f"margin-bottom={margin}", "--pdf-engine-opt=--disable-smart-shrinking", "--pdf-engine-opt=--disable-javascript", "--pdf-engine-opt=--no-images", "--pdf-engine-opt=--disable-local-file-access", "--filter=pandoc-mermaid", - "-f", "json", - f"/tmp/{randomFn}/input.js" + "-f", + "json", + f"/tmp/{randomFn}/input.js", ] # try: # # For some reasons, options are not passed correctly or not parsed correctly by wkhtmltopdf.. @@ -116,9 +127,9 @@ def convert(markdown, margin='3'): # except Exception as e: # print(e) - os.makedirs(f'/tmp/{randomFn}', exist_ok=True) + os.makedirs(f"/tmp/{randomFn}", exist_ok=True) # Write parsed file structure to be fed to the converter - with open(f'/tmp/{randomFn}/input.js', 'bw') as f: + with open(f"/tmp/{randomFn}/input.js", "bw") as f: configuration = pandoc.configure(read=True) if pandoc.utils.version_key(configuration["pandoc_types_version"]) < [1, 17]: json_ = pandoc.write_json_v1(doc) @@ -134,11 +145,11 @@ def convert(markdown, margin='3'): print(f"Command failed with error: {e}") # Read output and returns it - with open(f'/tmp/{randomFn}/output', 'br') as f: + with open(f"/tmp/{randomFn}/output", "br") as f: converted = f.read() # Clean up generated files - folderPath = f'/tmp/{randomFn}' + folderPath = f"/tmp/{randomFn}" try: shutil.rmtree(folderPath) print(f"Folder '{folderPath}' deleted successfully.") @@ -149,27 +160,27 @@ def convert(markdown, margin='3'): return base64.b64encode(converted).decode() + def handler(q=False): if q is False: return False request = json.loads(q) - if request.get('text'): - data = request['text'] + if request.get("text"): + data = request["text"] else: return False data = json.loads(data) - markdown = data.get('markdown') + markdown = data.get("markdown") try: - margin = '3' - if 'config' in request.get('config', []): - if request['config'].get('margin'): - margin = request['config'].get('margin') + margin = "3" + if "config" in request.get("config", []): + if request["config"].get("margin"): + margin = request["config"].get("margin") rendered = convert(markdown, margin=margin) except Exception as e: - rendered = f'Error: {e}' + rendered = f"Error: {e}" - r = {'results': [{'types': mispattributes['output'], - 'values':[rendered]}]} + r = {"results": [{"types": mispattributes["output"], "values": [rendered]}]} return r @@ -178,5 +189,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/countrycode.py b/misp_modules/modules/expansion/countrycode.py index 0bf740881..7d6b4fb1e 100755 --- a/misp_modules/modules/expansion/countrycode.py +++ b/misp_modules/modules/expansion/countrycode.py @@ -1,34 +1,39 @@ import json + import requests -misperrors = {'error': 'Error'} -mispattributes = {'input': ['hostname', 'domain']} +misperrors = {"error": "Error"} +mispattributes = {"input": ["hostname", "domain"]} # possible module-types: 'expansion', 'hover' or both moduleinfo = { - 'version': '1', - 'author': 'Hannah Ward', - 'description': 'Module to expand country codes.', - 'module-type': ['hover'], - 'name': 'Country Code', - 'logo': '', - 'requirements': [], - 'features': 'The module takes a domain or a hostname as input, and returns the country it belongs to.\n\nFor non country domains, a list of the most common possible extensions is used.', - 'references': [], - 'input': 'Hostname or domain attribute.', - 'output': 'Text with the country code the input belongs to.', + "version": "1", + "author": "Hannah Ward", + "description": "Module to expand country codes.", + "module-type": ["hover"], + "name": "Country Code", + "logo": "", + "requirements": [], + "features": ( + "The module takes a domain or a hostname as input, and returns the country it belongs to.\n\nFor non country" + " domains, a list of the most common possible extensions is used." + ), + "references": [], + "input": "Hostname or domain attribute.", + "output": "Text with the country code the input belongs to.", } # config fields that your code expects from the site admin moduleconfig = [] -common_tlds = {"com": "Commercial (Worldwide)", - "org": "Organisation (Worldwide)", - "net": "Network (Worldwide)", - "int": "International (Worldwide)", - "edu": "Education (Usually USA)", - "gov": "Government (USA)" - } +common_tlds = { + "com": "Commercial (Worldwide)", + "org": "Organisation (Worldwide)", + "net": "Network (Worldwide)", + "int": "International (Worldwide)", + "edu": "Education (Usually USA)", + "gov": "Government (USA)", +} def parse_country_code(extension): @@ -37,11 +42,11 @@ def parse_country_code(extension): codes = requests.get("http://www.geognos.com/api/en/countries/info/all.json").json() except Exception: return "http://www.geognos.com/api/en/countries/info/all.json not reachable" - if not codes.get('StatusMsg') or not codes["StatusMsg"] == "OK": - return 'Not able to get the countrycode references from http://www.geognos.com/api/en/countries/info/all.json' - for country in codes['Results'].values(): - if country['CountryCodes']['tld'] == extension: - return country['Name'] + if not codes.get("StatusMsg") or not codes["StatusMsg"] == "OK": + return "Not able to get the countrycode references from http://www.geognos.com/api/en/countries/info/all.json" + for country in codes["Results"].values(): + if country["CountryCodes"]["tld"] == extension: + return country["Name"] return "Unknown" @@ -56,7 +61,7 @@ def handler(q=False): # Check if it's a common, non country one val = common_tlds[ext] if ext in common_tlds.keys() else parse_country_code(ext) - r = {'results': [{'types': ['text'], 'values':[val]}]} + r = {"results": [{"types": ["text"], "values": [val]}]} return r @@ -65,5 +70,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/cpe.py b/misp_modules/modules/expansion/cpe.py index bd721fdc2..2cf50e4b0 100644 --- a/misp_modules/modules/expansion/cpe.py +++ b/misp_modules/modules/expansion/cpe.py @@ -1,131 +1,122 @@ import json + import requests -from . import check_input_attribute, standard_error_message from pymisp import MISPEvent, MISPObject -misperrors = {'error': 'Error'} -mispattributes = {'input': ['cpe'], 'format': 'misp_standard'} +from . import check_input_attribute, standard_error_message + +misperrors = {"error": "Error"} +mispattributes = {"input": ["cpe"], "format": "misp_standard"} moduleinfo = { - 'version': '2', - 'author': 'Christian Studer', - 'description': 'An expansion module to query the CVE search API with a cpe code to get its related vulnerabilities.', - 'module-type': ['expansion', 'hover'], - 'name': 'CPE Lookup', - 'logo': 'cve.png', - 'requirements': [], - 'features': 'The module takes a cpe attribute as input and queries the CVE search API to get its related vulnerabilities. \nThe list of vulnerabilities is then parsed and returned as vulnerability objects.\n\nUsers can use their own CVE search API url by defining a value to the custom_API_URL parameter. If no custom API url is given, the default vulnerability.circl.lu api url is used.\n\nIn order to limit the amount of data returned by CVE serach, users can also the limit parameter. With the limit set, the API returns only the requested number of vulnerabilities, sorted from the highest cvss score to the lowest one.', - 'references': ['https://vulnerability.circl.lu/api/'], - 'input': 'CPE attribute.', - 'output': 'The vulnerabilities related to the CPE.', + "version": "2", + "author": "Christian Studer", + "description": ( + "An expansion module to query the CVE search API with a cpe code to get its related vulnerabilities." + ), + "module-type": ["expansion", "hover"], + "name": "CPE Lookup", + "logo": "cve.png", + "requirements": [], + "features": ( + "The module takes a cpe attribute as input and queries the CVE search API to get its related vulnerabilities. " + " \nThe list of vulnerabilities is then parsed and returned as vulnerability objects.\n\nUsers can use their" + " own CVE search API url by defining a value to the custom_API_URL parameter. If no custom API url is given," + " the default vulnerability.circl.lu api url is used.\n\nIn order to limit the amount of data returned by CVE" + " serach, users can also the limit parameter. With the limit set, the API returns only the requested number of" + " vulnerabilities, sorted from the highest cvss score to the lowest one." + ), + "references": ["https://vulnerability.circl.lu/api/"], + "input": "CPE attribute.", + "output": "The vulnerabilities related to the CPE.", } moduleconfig = ["custom_API_URL", "limit"] -cveapi_url = 'https://cvepremium.circl.lu/api/query' +cveapi_url = "https://cvepremium.circl.lu/api/query" DEFAULT_LIMIT = 10 -class VulnerabilitiesParser(): +class VulnerabilitiesParser: def __init__(self, attribute): self.attribute = attribute self.misp_event = MISPEvent() self.misp_event.add_attribute(**attribute) self.vulnerability_mapping = { - 'id': { - 'type': 'vulnerability', - 'object_relation': 'id' - }, - 'summary': { - 'type': 'text', - 'object_relation': 'summary' - }, - 'vulnerable_configuration': { - 'type': 'cpe', - 'object_relation': 'vulnerable-configuration' - }, - 'vulnerable_configuration_cpe_2_2': { - 'type': 'cpe', - 'object_relation': 'vulnerable-configuration' - }, - 'Modified': { - 'type': 'datetime', - 'object_relation': 'modified' - }, - 'Published': { - 'type': 'datetime', - 'object_relation': 'published' + "id": {"type": "vulnerability", "object_relation": "id"}, + "summary": {"type": "text", "object_relation": "summary"}, + "vulnerable_configuration": { + "type": "cpe", + "object_relation": "vulnerable-configuration", }, - 'references': { - 'type': 'link', - 'object_relation': 'references' + "vulnerable_configuration_cpe_2_2": { + "type": "cpe", + "object_relation": "vulnerable-configuration", }, - 'cvss': { - 'type': 'float', - 'object_relation': 'cvss-score' - } + "Modified": {"type": "datetime", "object_relation": "modified"}, + "Published": {"type": "datetime", "object_relation": "published"}, + "references": {"type": "link", "object_relation": "references"}, + "cvss": {"type": "float", "object_relation": "cvss-score"}, } def parse_vulnerabilities(self, vulnerabilities): for vulnerability in vulnerabilities: - vulnerability_object = MISPObject('vulnerability') - for feature in ('id', 'summary', 'Modified', 'Published', 'cvss'): + vulnerability_object = MISPObject("vulnerability") + for feature in ("id", "summary", "Modified", "Published", "cvss"): if vulnerability.get(feature): - attribute = {'value': vulnerability[feature]} + attribute = {"value": vulnerability[feature]} attribute.update(self.vulnerability_mapping[feature]) vulnerability_object.add_attribute(**attribute) - if vulnerability.get('Published'): - vulnerability_object.add_attribute(**{ - 'type': 'text', - 'object_relation': 'state', - 'value': 'Published' - }) - for feature in ('references', 'vulnerable_configuration', 'vulnerable_configuration_cpe_2_2'): + if vulnerability.get("Published"): + vulnerability_object.add_attribute(**{"type": "text", "object_relation": "state", "value": "Published"}) + for feature in ( + "references", + "vulnerable_configuration", + "vulnerable_configuration_cpe_2_2", + ): if vulnerability.get(feature): for value in vulnerability[feature]: if isinstance(value, dict): - value = value['title'] - attribute = {'value': value} + value = value["title"] + attribute = {"value": value} attribute.update(self.vulnerability_mapping[feature]) vulnerability_object.add_attribute(**attribute) - vulnerability_object.add_reference(self.attribute['uuid'], 'related-to') + vulnerability_object.add_reference(self.attribute["uuid"], "related-to") self.misp_event.add_object(vulnerability_object) def get_result(self): event = json.loads(self.misp_event.to_json()) - results = {key: event[key] for key in ('Attribute', 'Object')} - return {'results': results} + results = {key: event[key] for key in ("Attribute", "Object")} + return {"results": results} def check_url(url): - return url if url.endswith('/') else f"{url}/" + return url if url.endswith("/") else f"{url}/" def handler(q=False): if q is False: return False request = json.loads(q) - if not request.get('attribute') or not check_input_attribute(request['attribute']): - return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} - attribute = request['attribute'] - if attribute.get('type') != 'cpe': - return {'error': 'Wrong input attribute type.'} - config = request['config'] - url = check_url(config['custom_API_URL']) if config.get('custom_API_URL') else cveapi_url - limit = int(config['limit']) if config.get('limit') else DEFAULT_LIMIT + if not request.get("attribute") or not check_input_attribute(request["attribute"]): + return {"error": f"{standard_error_message}, which should contain at least a type, a value and an uuid."} + attribute = request["attribute"] + if attribute.get("type") != "cpe": + return {"error": "Wrong input attribute type."} + config = request["config"] + url = check_url(config["custom_API_URL"]) if config.get("custom_API_URL") else cveapi_url + limit = int(config["limit"]) if config.get("limit") else DEFAULT_LIMIT params = { "retrieve": "cves", - "dict_filter": { - "vulnerable_configuration": attribute['value'] - }, + "dict_filter": {"vulnerable_configuration": attribute["value"]}, "limit": limit, "sort": "cvss", - "sort_dir": "DESC" + "sort_dir": "DESC", } response = requests.post(url, json=params) if response.status_code == 200: - vulnerabilities = response.json()['data'] + vulnerabilities = response.json()["data"] if not vulnerabilities: - return {'error': 'No related vulnerability for this CPE.'} + return {"error": "No related vulnerability for this CPE."} else: - return {'error': 'API not accessible.'} + return {"error": "API not accessible."} parser = VulnerabilitiesParser(attribute) parser.parse_vulnerabilities(vulnerabilities) return parser.get_result() @@ -136,5 +127,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/crowdsec.py b/misp_modules/modules/expansion/crowdsec.py index 983ac88b3..6731ecf55 100644 --- a/misp_modules/modules/expansion/crowdsec.py +++ b/misp_modules/modules/expansion/crowdsec.py @@ -16,9 +16,14 @@ "name": "CrowdSec CTI", "logo": "crowdsec.png", "requirements": [ - "A CrowdSec CTI API key. Get yours by following https://docs.crowdsec.net/docs/cti_api/getting_started/#getting-an-api-key" + "A CrowdSec CTI API key. Get yours by following" + " https://docs.crowdsec.net/docs/cti_api/getting_started/#getting-an-api-key" ], - "features": "This module enables IP lookup from CrowdSec CTI API. It provides information about the IP, such as what kind of attacks it has been participant of as seen by CrowdSec's network. It also includes enrichment by CrowdSec like background noise score, aggressivity over time etc.", + "features": ( + "This module enables IP lookup from CrowdSec CTI API. It provides information about the IP, such as what kind" + " of attacks it has been participant of as seen by CrowdSec's network. It also includes enrichment by CrowdSec" + " like background noise score, aggressivity over time etc." + ), "references": [ "https://www.crowdsec.net/", "https://docs.crowdsec.net/docs/cti_api/getting_started", @@ -49,14 +54,10 @@ def handler(q=False): return {"error": "Missing CrowdSec API key"} if not request.get("attribute") or not check_input_attribute(request["attribute"]): - return { - "error": f"{standard_error_message}, which should contain at least a type, a value and an uuid." - } + return {"error": f"{standard_error_message}, which should contain at least a type, a value and an uuid."} if request["attribute"].get("type") not in mispattributes["input"]: - return { - "error": f"Wrong input type. Please choose on of the following: {', '.join(mispattributes['input'])}" - } + return {"error": f"Wrong input type. Please choose on of the following: {', '.join(mispattributes['input'])}"} return _handler_v2(request) @@ -81,9 +82,7 @@ def _handler_v2(request_data): try: ipaddress.ip_address(ip) except ValueError: - return { - "error": f"IP ({ip}) is not valid for calling CrowdSec CTI. Please provide a valid IP address." - } + return {"error": f"IP ({ip}) is not valid for calling CrowdSec CTI. Please provide a valid IP address."} crowdsec_cti = requests.get( f"https://cti.api.crowdsec.net/v2/smoke/{ip}", @@ -97,12 +96,8 @@ def _handler_v2(request_data): add_reputation_tag = _get_boolean_config(request_data, "add_reputation_tag", True) add_behavior_tag = _get_boolean_config(request_data, "add_behavior_tag", True) - add_classification_tag = _get_boolean_config( - request_data, "add_classification_tag", True - ) - add_mitre_technique_tag = _get_boolean_config( - request_data, "add_mitre_technique_tag", True - ) + add_classification_tag = _get_boolean_config(request_data, "add_classification_tag", True) + add_mitre_technique_tag = _get_boolean_config(request_data, "add_mitre_technique_tag", True) add_cve_tag = _get_boolean_config(request_data, "add_cve_tag", True) misp_event = MISPEvent() @@ -119,36 +114,20 @@ def _handler_v2(request_data): tag = f'crowdsec:reputation="{reputation}"' ip_attribute.add_tag(tag) crowdsec_context_object.add_attribute("ip-range", crowdsec_cti["ip_range"]) - crowdsec_context_object.add_attribute( - "ip-range-score", crowdsec_cti["ip_range_score"] - ) - crowdsec_context_object.add_attribute( - "country", get_country_name_from_alpha_2(crowdsec_cti["location"]["country"]) - ) - crowdsec_context_object.add_attribute( - "country-code", crowdsec_cti["location"]["country"] - ) + crowdsec_context_object.add_attribute("ip-range-score", crowdsec_cti["ip_range_score"]) + crowdsec_context_object.add_attribute("country", get_country_name_from_alpha_2(crowdsec_cti["location"]["country"])) + crowdsec_context_object.add_attribute("country-code", crowdsec_cti["location"]["country"]) if crowdsec_cti["location"].get("city"): crowdsec_context_object.add_attribute("city", crowdsec_cti["location"]["city"]) - crowdsec_context_object.add_attribute( - "latitude", crowdsec_cti["location"]["latitude"] - ) - crowdsec_context_object.add_attribute( - "longitude", crowdsec_cti["location"]["longitude"] - ) + crowdsec_context_object.add_attribute("latitude", crowdsec_cti["location"]["latitude"]) + crowdsec_context_object.add_attribute("longitude", crowdsec_cti["location"]["longitude"]) crowdsec_context_object.add_attribute("as-name", crowdsec_cti["as_name"]) crowdsec_context_object.add_attribute("as-num", crowdsec_cti["as_num"]) if crowdsec_cti.get("reverse_dns") is not None: - crowdsec_context_object.add_attribute( - "reverse-dns", crowdsec_cti["reverse_dns"] - ) - crowdsec_context_object.add_attribute( - "background-noise", crowdsec_cti["background_noise_score"] - ) + crowdsec_context_object.add_attribute("reverse-dns", crowdsec_cti["reverse_dns"]) + crowdsec_context_object.add_attribute("background-noise", crowdsec_cti["background_noise_score"]) for behavior in crowdsec_cti["behaviors"]: - crowdsec_context_object.add_attribute( - "behaviors", behavior["label"], comment=behavior["description"] - ) + crowdsec_context_object.add_attribute("behaviors", behavior["label"], comment=behavior["description"]) if add_behavior_tag: tag = f'crowdsec:behavior="{behavior["name"]}"' ip_attribute.add_tag(tag) @@ -178,9 +157,7 @@ def _handler_v2(request_data): for feature, values in crowdsec_cti["classifications"].items(): field = feature[:-1] for value in values: - crowdsec_context_object.add_attribute( - feature, value["label"], comment=value["description"] - ) + crowdsec_context_object.add_attribute(feature, value["label"], comment=value["description"]) if add_classification_tag: tag = f'crowdsec:{field}="{value["name"]}"' ip_attribute.add_tag(tag) @@ -193,13 +170,9 @@ def _handler_v2(request_data): ) crowdsec_context_object.add_attribute( "target-countries", - ", ".join( - map(get_country_name_from_alpha_2, crowdsec_cti["target_countries"].keys()) - ), - ) - crowdsec_context_object.add_attribute( - "trust", crowdsec_cti["scores"]["overall"]["trust"] + ", ".join(map(get_country_name_from_alpha_2, crowdsec_cti["target_countries"].keys())), ) + crowdsec_context_object.add_attribute("trust", crowdsec_cti["scores"]["overall"]["trust"]) scores = [] for time_period, indicators in crowdsec_cti["scores"].items(): tp = " ".join(map(str.capitalize, time_period.split("_"))) @@ -213,11 +186,7 @@ def _handler_v2(request_data): misp_event.add_object(crowdsec_context_object) event = json.loads(misp_event.to_json()) - results = { - key: event[key] - for key in ("Attribute", "Object") - if (key in event and event[key]) - } + results = {key: event[key] for key in ("Attribute", "Object") if (key in event and event[key])} return {"results": results} diff --git a/misp_modules/modules/expansion/crowdstrike_falcon.py b/misp_modules/modules/expansion/crowdstrike_falcon.py index 43467d3e8..395fd8456 100755 --- a/misp_modules/modules/expansion/crowdstrike_falcon.py +++ b/misp_modules/modules/expansion/crowdstrike_falcon.py @@ -1,72 +1,107 @@ import json -from . import check_input_attribute, standard_error_message + from falconpy import Intel from pymisp import MISPAttribute, MISPEvent +from . import check_input_attribute, standard_error_message + moduleinfo = { - 'version': '0.2', - 'author': 'Christophe Vandeplas', - 'description': 'Module to query CrowdStrike Falcon.', - 'module-type': ['expansion', 'hover'], - 'name': 'CrowdStrike Falcon', - 'logo': 'crowdstrike.png', - 'requirements': ['A CrowdStrike API access (API id & key)'], - 'features': 'This module takes a MISP attribute as input to query a CrowdStrike Falcon API. The API returns then the result of the query with some types we map into compatible types we add as MISP attributes.\n\nPlease note that composite attributes composed by at least one of the input types mentionned below (domains, IPs, hostnames) are also supported.', - 'references': ['https://www.crowdstrike.com/products/crowdstrike-falcon-faq/'], - 'input': 'A MISP attribute included in the following list:\n- domain\n- email-attachment\n- email-dst\n- email-reply-to\n- email-src\n- email-subject\n- filename\n- hostname\n- ip-src\n- ip-dst\n- md5\n- mutex\n- regkey\n- sha1\n- sha256\n- uri\n- url\n- user-agent\n- whois-registrant-email\n- x509-fingerprint-md5', - 'output': 'MISP attributes mapped after the CrowdStrike API has been queried, included in the following list:\n- hostname\n- email-src\n- email-subject\n- filename\n- md5\n- sha1\n- sha256\n- ip-dst\n- ip-dst\n- mutex\n- regkey\n- url\n- user-agent\n- x509-fingerprint-md5', + "version": "0.2", + "author": "Christophe Vandeplas", + "description": "Module to query CrowdStrike Falcon.", + "module-type": ["expansion", "hover"], + "name": "CrowdStrike Falcon", + "logo": "crowdstrike.png", + "requirements": ["A CrowdStrike API access (API id & key)"], + "features": ( + "This module takes a MISP attribute as input to query a CrowdStrike Falcon API. The API returns then the result" + " of the query with some types we map into compatible types we add as MISP attributes.\n\nPlease note that" + " composite attributes composed by at least one of the input types mentionned below (domains, IPs, hostnames)" + " are also supported." + ), + "references": ["https://www.crowdstrike.com/products/crowdstrike-falcon-faq/"], + "input": ( + "A MISP attribute included in the following list:\n- domain\n- email-attachment\n- email-dst\n-" + " email-reply-to\n- email-src\n- email-subject\n- filename\n- hostname\n- ip-src\n- ip-dst\n- md5\n- mutex\n-" + " regkey\n- sha1\n- sha256\n- uri\n- url\n- user-agent\n- whois-registrant-email\n- x509-fingerprint-md5" + ), + "output": ( + "MISP attributes mapped after the CrowdStrike API has been queried, included in the following list:\n-" + " hostname\n- email-src\n- email-subject\n- filename\n- md5\n- sha1\n- sha256\n- ip-dst\n- ip-dst\n- mutex\n-" + " regkey\n- url\n- user-agent\n- x509-fingerprint-md5" + ), } -moduleconfig = ['api_id', 'apikey'] -misperrors = {'error': 'Error'} -misp_type_in = ['domain', 'email-attachment', 'email-dst', 'email-reply-to', 'email-src', 'email-subject', - 'filename', 'hostname', 'ip', 'ip-src', 'ip-dst', 'md5', 'mutex', 'regkey', 'sha1', 'sha256', 'uri', 'url', - 'user-agent', 'whois-registrant-email', 'x509-fingerprint-md5'] +moduleconfig = ["api_id", "apikey"] +misperrors = {"error": "Error"} +misp_type_in = [ + "domain", + "email-attachment", + "email-dst", + "email-reply-to", + "email-src", + "email-subject", + "filename", + "hostname", + "ip", + "ip-src", + "ip-dst", + "md5", + "mutex", + "regkey", + "sha1", + "sha256", + "uri", + "url", + "user-agent", + "whois-registrant-email", + "x509-fingerprint-md5", +] mapping_out = { # mapping between the MISP attributes type and the compatible CrowdStrike indicator types. - 'domain': {'type': 'hostname', 'to_ids': True}, - 'email_address': {'type': 'email-src', 'to_ids': True}, - 'email_subject': {'type': 'email-subject', 'to_ids': True}, - 'file_name': {'type': 'filename', 'to_ids': True}, - 'hash_md5': {'type': 'md5', 'to_ids': True}, - 'hash_sha1': {'type': 'sha1', 'to_ids': True}, - 'hash_sha256': {'type': 'sha256', 'to_ids': True}, - 'ip_address': {'type': 'ip-dst', 'to_ids': True}, - 'ip_address_block': {'type': 'ip-dst', 'to_ids': True}, - 'mutex_name': {'type': 'mutex', 'to_ids': True}, - 'registry': {'type': 'regkey', 'to_ids': True}, - 'url': {'type': 'url', 'to_ids': True}, - 'user_agent': {'type': 'user-agent', 'to_ids': True}, - 'x509_serial': {'type': 'x509-fingerprint-md5', 'to_ids': True}, - - 'actors': {'type': 'threat-actor', 'category': 'Attribution'}, - 'malware_families': {'type': 'text', 'category': 'Attribution'} + "domain": {"type": "hostname", "to_ids": True}, + "email_address": {"type": "email-src", "to_ids": True}, + "email_subject": {"type": "email-subject", "to_ids": True}, + "file_name": {"type": "filename", "to_ids": True}, + "hash_md5": {"type": "md5", "to_ids": True}, + "hash_sha1": {"type": "sha1", "to_ids": True}, + "hash_sha256": {"type": "sha256", "to_ids": True}, + "ip_address": {"type": "ip-dst", "to_ids": True}, + "ip_address_block": {"type": "ip-dst", "to_ids": True}, + "mutex_name": {"type": "mutex", "to_ids": True}, + "registry": {"type": "regkey", "to_ids": True}, + "url": {"type": "url", "to_ids": True}, + "user_agent": {"type": "user-agent", "to_ids": True}, + "x509_serial": {"type": "x509-fingerprint-md5", "to_ids": True}, + "actors": {"type": "threat-actor", "category": "Attribution"}, + "malware_families": {"type": "text", "category": "Attribution"}, } -misp_type_out = [item['type'] for item in mapping_out.values()] -mispattributes = {'input': misp_type_in, 'format': 'misp_standard'} +misp_type_out = [item["type"] for item in mapping_out.values()] +mispattributes = {"input": misp_type_in, "format": "misp_standard"} + def handler(q=False): if q is False: return False request = json.loads(q) - #validate CrowdStrike params - if (request.get('config')): - if (request['config'].get('apikey') is None): - misperrors['error'] = 'CrowdStrike apikey is missing' + # validate CrowdStrike params + if request.get("config"): + if request["config"].get("apikey") is None: + misperrors["error"] = "CrowdStrike apikey is missing" return misperrors - if (request['config'].get('api_id') is None): - misperrors['error'] = 'CrowdStrike api_id is missing' + if request["config"].get("api_id") is None: + misperrors["error"] = "CrowdStrike api_id is missing" return misperrors - #validate attribute - if not request.get('attribute') or not check_input_attribute(request['attribute']): - return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} - attribute = request.get('attribute') - if not any(input_type == attribute.get('type') for input_type in misp_type_in): - return {'error': 'Unsupported attribute type.'} + # validate attribute + if not request.get("attribute") or not check_input_attribute(request["attribute"]): + return {"error": f"{standard_error_message}, which should contain at least a type, a value and an uuid."} + attribute = request.get("attribute") + if not any(input_type == attribute.get("type") for input_type in misp_type_in): + return {"error": "Unsupported attribute type."} - client = CSIntelAPI(request['config']['api_id'], request['config']['apikey']) + client = CSIntelAPI(request["config"]["api_id"], request["config"]["apikey"]) attribute = MISPAttribute() - attribute.from_dict(**request.get('attribute') ) + attribute.from_dict(**request.get("attribute")) r = {"results": []} valid_type = False @@ -74,15 +109,15 @@ def handler(q=False): for k in misp_type_in: if attribute.type == k: # map the MISP type to the CrowdStrike type - r['results'].append(lookup_indicator(client, attribute)) + r["results"].append(lookup_indicator(client, attribute)) valid_type = True except Exception as e: - return {'error': f"{e}"} + return {"error": f"{e}"} if not valid_type: - misperrors['error'] = "Unsupported attributes type" + misperrors["error"] = "Unsupported attributes type" return misperrors - return {'results': r.get('results').pop()} + return {"results": r.get("results").pop()} def lookup_indicator(client, ref_attribute): @@ -90,40 +125,41 @@ def lookup_indicator(client, ref_attribute): misp_event = MISPEvent() misp_event.add_attribute(**ref_attribute) - for item in result.get('resources', []): - for relation in item.get('relations'): - if mapping_out.get(relation.get('type')): - r = mapping_out[relation.get('type')].copy() - r['value'] = relation.get('indicator') + for item in result.get("resources", []): + for relation in item.get("relations"): + if mapping_out.get(relation.get("type")): + r = mapping_out[relation.get("type")].copy() + r["value"] = relation.get("indicator") attribute = MISPAttribute() attribute.from_dict(**r) misp_event.add_attribute(**attribute) - for actor in item.get('actors'): - r = mapping_out.get('actors').copy() - r['value'] = actor + for actor in item.get("actors"): + r = mapping_out.get("actors").copy() + r["value"] = actor attribute = MISPAttribute() attribute.from_dict(**r) misp_event.add_attribute(**attribute) - if item.get('malware_families'): - r = mapping_out.get('malware_families').copy() - r['value'] = f"malware_families: {' | '.join(item.get('malware_families'))}" + if item.get("malware_families"): + r = mapping_out.get("malware_families").copy() + r["value"] = f"malware_families: {' | '.join(item.get('malware_families'))}" attribute = MISPAttribute() attribute.from_dict(**r) misp_event.add_attribute(**attribute) event = json.loads(misp_event.to_json()) - return {'Object': event.get('Object', []), 'Attribute': event.get('Attribute', [])} + return {"Object": event.get("Object", []), "Attribute": event.get("Attribute", [])} + def introspection(): return mispattributes def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo -class CSIntelAPI(): +class CSIntelAPI: def __init__(self, custid=None, custkey=None): # customer id and key should be passed when obj is created self.falcon = Intel(client_id=custid, client_secret=custkey) @@ -131,18 +167,18 @@ def __init__(self, custid=None, custkey=None): def search_indicator(self, query): r = self.falcon.query_indicator_entities(q=query) # 400 - bad request - if r.get('status_code') == 400: - raise Exception('HTTP Error 400 - Bad request.') + if r.get("status_code") == 400: + raise Exception("HTTP Error 400 - Bad request.") # 404 - oh shit - if r.get('status_code') == 404: - raise Exception('HTTP Error 404 - awww snap.') + if r.get("status_code") == 404: + raise Exception("HTTP Error 404 - awww snap.") # catch all? - if r.get('status_code') != 200: - raise Exception('HTTP Error: ' + str(r.get('status_code'))) + if r.get("status_code") != 200: + raise Exception("HTTP Error: " + str(r.get("status_code"))) - if len(r.get('body').get('errors')): - raise Exception('API Error: ' + ' | '.join(r.get('body').get('errors'))) + if len(r.get("body").get("errors")): + raise Exception("API Error: " + " | ".join(r.get("body").get("errors"))) - return r.get('body', {}) \ No newline at end of file + return r.get("body", {}) diff --git a/misp_modules/modules/expansion/cuckoo_submit.py b/misp_modules/modules/expansion/cuckoo_submit.py index 912697447..c6bf8680a 100644 --- a/misp_modules/modules/expansion/cuckoo_submit.py +++ b/misp_modules/modules/expansion/cuckoo_submit.py @@ -2,47 +2,49 @@ import io import json import logging -import requests import sys import urllib.parse import zipfile +import requests from requests.exceptions import RequestException log = logging.getLogger("cuckoo_submit") log.setLevel(logging.DEBUG) sh = logging.StreamHandler(sys.stdout) sh.setLevel(logging.DEBUG) -fmt = logging.Formatter( - "%(asctime)s - %(name)s - %(levelname)s - %(message)s" -) +fmt = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") sh.setFormatter(fmt) log.addHandler(sh) moduleinfo = { - 'version': '0.1', - 'author': 'Evert Kors', - 'description': 'Submit files and URLs to Cuckoo Sandbox', - 'module-type': ['expansion', 'hover'], - 'name': 'Cuckoo Submit', - 'logo': 'cuckoo.png', - 'requirements': ['Access to a Cuckoo Sandbox API and an API key if the API requires it. (api_url and api_key)'], - 'features': 'The module takes a malware-sample, attachment, url or domain and submits it to Cuckoo Sandbox.\n The returned task id can be used to retrieve results when the analysis completed.', - 'references': ['https://cuckoosandbox.org/', 'https://cuckoo.sh/docs/'], - 'input': 'A malware-sample or attachment for files. A url or domain for URLs.', - 'output': "A text field containing 'Cuckoo task id: '", + "version": "0.1", + "author": "Evert Kors", + "description": "Submit files and URLs to Cuckoo Sandbox", + "module-type": ["expansion", "hover"], + "name": "Cuckoo Submit", + "logo": "cuckoo.png", + "requirements": ["Access to a Cuckoo Sandbox API and an API key if the API requires it. (api_url and api_key)"], + "features": ( + "The module takes a malware-sample, attachment, url or domain and submits it to Cuckoo Sandbox.\n The returned" + " task id can be used to retrieve results when the analysis completed." + ), + "references": ["https://cuckoosandbox.org/", "https://cuckoo.sh/docs/"], + "input": "A malware-sample or attachment for files. A url or domain for URLs.", + "output": "A text field containing 'Cuckoo task id: '", } misperrors = {"error": "Error"} moduleconfig = ["api_url", "api_key"] mispattributes = { "input": ["attachment", "malware-sample", "url", "domain"], - "output": ["text"] + "output": ["text"], } class APIKeyError(RequestException): """Raised if the Cuckoo API returns a 401. This means no or an invalid bearer token was supplied.""" + pass @@ -56,15 +58,14 @@ def __init__(self, api_url, api_key=""): self.api_url = api_url def _post_api(self, endpoint, files=None, data={}): - data.update({ - "owner": "MISP" - }) + data.update({"owner": "MISP"}) try: response = requests.post( urllib.parse.urljoin(self.api_url, endpoint), - files=files, data=data, - headers={"Authorization": "Bearer {}".format(self.api_key)} + files=files, + data=data, + headers={"Authorization": "Bearer {}".format(self.api_key)}, ) except RequestException as e: log.error("Failed to submit sample to Cuckoo Sandbox. %s", e) @@ -80,18 +81,14 @@ def _post_api(self, endpoint, files=None, data={}): return response.json() def create_task(self, filename, fp): - response = self._post_api( - "/tasks/create/file", files={"file": (filename, fp)} - ) + response = self._post_api("/tasks/create/file", files={"file": (filename, fp)}) if not response: return False return response["task_id"] def create_url(self, url): - response = self._post_api( - "/tasks/create/url", data={"url": url} - ) + response = self._post_api("/tasks/create/url", data={"url": url}) if not response: return False @@ -134,9 +131,7 @@ def handler(q=False): task_id = cuckoo_api.create_url(url) elif data and filename: log.debug("Submitting file to Cuckoo Sandbox %s", api_url) - task_id = cuckoo_api.create_task( - filename=filename, fp=io.BytesIO(data) - ) + task_id = cuckoo_api.create_task(filename=filename, fp=io.BytesIO(data)) except APIKeyError as e: misperrors["error"] = "Failed to submit to Cuckoo: {}".format(e) return misperrors @@ -145,11 +140,7 @@ def handler(q=False): misperrors["error"] = "File or URL submission failed" return misperrors - return { - "results": [ - {"types": "text", "values": "Cuckoo task id: {}".format(task_id)} - ] - } + return {"results": [{"types": "text", "values": "Cuckoo task id: {}".format(task_id)}]} def introspection(): diff --git a/misp_modules/modules/expansion/cve.py b/misp_modules/modules/expansion/cve.py index 6321d2836..8455a4121 100755 --- a/misp_modules/modules/expansion/cve.py +++ b/misp_modules/modules/expansion/cve.py @@ -1,47 +1,49 @@ import json + import requests + from . import check_input_attribute, standard_error_message from ._vulnerability_parser.vulnerability_parser import VulnerabilityLookupParser -misperrors = {'error': 'Error'} -mispattributes = {'input': ['vulnerability'], 'format': 'misp_standard'} +misperrors = {"error": "Error"} +mispattributes = {"input": ["vulnerability"], "format": "misp_standard"} moduleinfo = { - 'version': '2', - 'author': 'Alexandre Dulaunoy', - 'description': 'An expansion hover module to expand information about CVE id.', - 'module-type': ['expansion', 'hover'], - 'name': 'CVE Lookup', - 'logo': 'vulnerability_lookyp.png', - 'requirements': [], - 'features': 'The module takes a vulnerability attribute as input and queries Vulnerability Lookup to get additional information based on the Vulnerability ID.', - 'references': ['https://cve.circl.lu/', 'https://cve.mitre.org/'], - 'input': 'Vulnerability attribute.', - 'output': 'Additional information on the vulnerability, gathered from the Vulnerability Lookup API.', + "version": "2", + "author": "Alexandre Dulaunoy", + "description": "An expansion hover module to expand information about CVE id.", + "module-type": ["expansion", "hover"], + "name": "CVE Lookup", + "logo": "vulnerability_lookyp.png", + "requirements": [], + "features": ( + "The module takes a vulnerability attribute as input and queries Vulnerability Lookup to get additional" + " information based on the Vulnerability ID." + ), + "references": ["https://cve.circl.lu/", "https://cve.mitre.org/"], + "input": "Vulnerability attribute.", + "output": "Additional information on the vulnerability, gathered from the Vulnerability Lookup API.", } -api_url = 'https://cve.circl.lu' +api_url = "https://cve.circl.lu" def handler(q=False): if q is False: return False request = json.loads(q) - if not check_input_attribute(request.get('attribute', {})): - return { - 'error': f'{standard_error_message}, which should contain ' - 'at least a type, a value and an UUID.' - } - attribute = request['attribute'] - if attribute.get('type') != 'vulnerability': - return {'error': 'The attribute type should be "vulnerability".'} + if not check_input_attribute(request.get("attribute", {})): + return {"error": f"{standard_error_message}, which should contain at least a type, a value and an UUID."} + attribute = request["attribute"] + if attribute.get("type") != "vulnerability": + return {"error": 'The attribute type should be "vulnerability".'} lookup = requests.get(f"{api_url}/api/vulnerability/{attribute['value']}") if lookup.status_code == 200: vulnerability = lookup.json() if not vulnerability: - return {'error': 'Non existing vulnerability ID.'} + return {"error": "Non existing vulnerability ID."} else: - return {'error': 'Vulnerability Lookup API not accessible.'} + return {"error": "Vulnerability Lookup API not accessible."} parser = VulnerabilityLookupParser(attribute, api_url) - parser.parser_lookup_result(vulnerability) + parser.parse_lookup_result(vulnerability) return parser.get_results() diff --git a/misp_modules/modules/expansion/cve_advanced.py b/misp_modules/modules/expansion/cve_advanced.py index b1e4c8413..e051b0eb1 100644 --- a/misp_modules/modules/expansion/cve_advanced.py +++ b/misp_modules/modules/expansion/cve_advanced.py @@ -1,29 +1,42 @@ import json -import requests -from . import check_input_attribute, standard_error_message from collections import defaultdict + +import requests from pymisp import MISPAttribute, MISPEvent, MISPObject -misperrors = {'error': 'Error'} -mispattributes = {'input': ['vulnerability'], 'format': 'misp_standard'} +from . import check_input_attribute, standard_error_message + +misperrors = {"error": "Error"} +mispattributes = {"input": ["vulnerability"], "format": "misp_standard"} moduleinfo = { - 'version': '2', - 'author': 'Christian Studer', - 'description': 'An expansion module to query the CIRCL CVE search API for more information about a vulnerability (CVE).', - 'module-type': ['expansion', 'hover'], - 'name': 'CVE Advanced Lookup', - 'logo': 'cve.png', - 'requirements': [], - 'features': 'The module takes a vulnerability attribute as input and queries the CIRCL CVE search API to gather additional information.\n\nThe result of the query is then parsed to return additional information about the vulnerability, like its cvss score or some references, as well as the potential related weaknesses and attack patterns.\n\nThe vulnerability additional data is returned in a vulnerability MISP object, and the related additional information are put into weakness and attack-pattern MISP objects.', - 'references': ['https://vulnerability.circl.lu', 'https://cve/mitre.org/'], - 'input': 'Vulnerability attribute.', - 'output': 'Additional information about the vulnerability, such as its cvss score, some references, or the related weaknesses and attack patterns.', + "version": "2", + "author": "Christian Studer", + "description": ( + "An expansion module to query the CIRCL CVE search API for more information about a vulnerability (CVE)." + ), + "module-type": ["expansion", "hover"], + "name": "CVE Advanced Lookup", + "logo": "cve.png", + "requirements": [], + "features": ( + "The module takes a vulnerability attribute as input and queries the CIRCL CVE search API to gather additional" + " information.\n\nThe result of the query is then parsed to return additional information about the" + " vulnerability, like its cvss score or some references, as well as the potential related weaknesses and attack" + " patterns.\n\nThe vulnerability additional data is returned in a vulnerability MISP object, and the related" + " additional information are put into weakness and attack-pattern MISP objects." + ), + "references": ["https://vulnerability.circl.lu", "https://cve/mitre.org/"], + "input": "Vulnerability attribute.", + "output": ( + "Additional information about the vulnerability, such as its cvss score, some references, or the related" + " weaknesses and attack patterns." + ), } moduleconfig = ["custom_API"] -cveapi_url = 'https://cvepremium.circl.lu/api/' +cveapi_url = "https://cvepremium.circl.lu/api/" -class VulnerabilityParser(): +class VulnerabilityParser: def __init__(self, attribute, api_url): misp_attribute = MISPAttribute() misp_attribute.from_dict(**attribute) @@ -33,20 +46,24 @@ def __init__(self, attribute, api_url): self.__misp_event = misp_event self.__api_url = api_url self.references = defaultdict(list) - self.__capec_features = ('id', 'name', 'summary', 'prerequisites', 'solutions') + self.__capec_features = ("id", "name", "summary", "prerequisites", "solutions") self.__vulnerability_mapping = { - 'id': 'id', 'summary': 'summary', - 'Modified': 'modified', 'cvss3': 'cvss-score', - 'cvss3-vector': 'cvss-string' + "id": "id", + "summary": "summary", + "Modified": "modified", + "cvss3": "cvss-score", + "cvss3-vector": "cvss-string", } self.__vulnerability_multiple_mapping = { - 'vulnerable_configuration': 'vulnerable-configuration', - 'vulnerable_configuration_cpe_2_2': 'vulnerable-configuration', - 'references': 'references' + "vulnerable_configuration": "vulnerable-configuration", + "vulnerable_configuration_cpe_2_2": "vulnerable-configuration", + "references": "references", } self.__weakness_mapping = { - 'name': 'name', 'description_summary': 'description', - 'status': 'status', 'weaknessabs': 'weakness-abs' + "name": "name", + "description_summary": "description", + "status": "status", + "weaknessabs": "weakness-abs", } @property @@ -81,29 +98,32 @@ def get_result(self): if self.references: self.__build_references() event = json.loads(self.misp_event.to_json()) - results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} - return {'results': results} + results = {key: event[key] for key in ("Attribute", "Object") if (key in event and event[key])} + return {"results": results} def parse_vulnerability_information(self, vulnerability): - vulnerability_object = MISPObject('vulnerability') + vulnerability_object = MISPObject("vulnerability") for feature, relation in self.vulnerability_mapping.items(): if vulnerability.get(feature): vulnerability_object.add_attribute(relation, vulnerability[feature]) - if 'Published' in vulnerability: - vulnerability_object.add_attribute('published', vulnerability['Published']) - vulnerability_object.add_attribute('state', 'Published') + if "Published" in vulnerability: + vulnerability_object.add_attribute("published", vulnerability["Published"]) + vulnerability_object.add_attribute("state", "Published") for feature, relation in self.vulnerability_multiple_mapping.items(): if feature in vulnerability: for value in vulnerability[feature]: if isinstance(value, dict): - value = value['title'] + value = value["title"] vulnerability_object.add_attribute(relation, value) - vulnerability_object.add_reference(self.misp_attribute.uuid, 'related-to') + vulnerability_object.add_reference(self.misp_attribute.uuid, "related-to") self.misp_event.add_object(vulnerability_object) - if 'cwe' in vulnerability and vulnerability['cwe'] not in ('Unknown', 'NVD-CWE-noinfo'): - self.__parse_weakness(vulnerability['cwe'], vulnerability_object.uuid) - if 'capec' in vulnerability: - self.__parse_capec(vulnerability['capec'], vulnerability_object.uuid) + if "cwe" in vulnerability and vulnerability["cwe"] not in ( + "Unknown", + "NVD-CWE-noinfo", + ): + self.__parse_weakness(vulnerability["cwe"], vulnerability_object.uuid) + if "capec" in vulnerability: + self.__parse_capec(vulnerability["capec"], vulnerability_object.uuid) def __build_references(self): for object_uuid, references in self.references.items(): @@ -115,62 +135,62 @@ def __build_references(self): def __parse_capec(self, capec_values, vulnerability_uuid): for capec in capec_values: - capec_object = MISPObject('attack-pattern') + capec_object = MISPObject("attack-pattern") for feature in self.capec_features: capec_object.add_attribute(feature, capec[feature]) - for related_weakness in capec['related_weakness']: - capec_object.add_attribute('related-weakness', f"CWE-{related_weakness}") + for related_weakness in capec["related_weakness"]: + capec_object.add_attribute("related-weakness", f"CWE-{related_weakness}") self.misp_event.add_object(capec_object) self.references[vulnerability_uuid].append( { - 'referenced_uuid': capec_object.uuid, - 'relationship_type': 'targeted-by' + "referenced_uuid": capec_object.uuid, + "relationship_type": "targeted-by", } ) def __parse_weakness(self, cwe_value, vulnerability_uuid): - cwe_string, cwe_id = cwe_value.split('-')[:2] - cwe = requests.get(f'{self.api_url}cwe/{cwe_id}') + cwe_string, cwe_id = cwe_value.split("-")[:2] + cwe = requests.get(f"{self.api_url}cwe/{cwe_id}") if cwe.status_code == 200: cwe = cwe.json() - weakness_object = MISPObject('weakness') - weakness_object.add_attribute('id', f'{cwe_string}-{cwe_id}') + weakness_object = MISPObject("weakness") + weakness_object.add_attribute("id", f"{cwe_string}-{cwe_id}") for feature, relation in self.weakness_mapping.items(): if cwe.get(feature): weakness_object.add_attribute(relation, cwe[feature]) self.misp_event.add_object(weakness_object) self.references[vulnerability_uuid].append( { - 'referenced_uuid': weakness_object.uuid, - 'relationship_type': 'weakened-by' + "referenced_uuid": weakness_object.uuid, + "relationship_type": "weakened-by", } ) def check_url(url): - return f"{url}/" if not url.endswith('/') else url + return f"{url}/" if not url.endswith("/") else url def handler(q=False): if q is False: return False request = json.loads(q) - if not request.get('attribute') or not check_input_attribute(request['attribute']): - return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} - attribute = request['attribute'] - if attribute.get('type') != 'vulnerability': - misperrors['error'] = 'Vulnerability id missing.' + if not request.get("attribute") or not check_input_attribute(request["attribute"]): + return {"error": f"{standard_error_message}, which should contain at least a type, a value and an uuid."} + attribute = request["attribute"] + if attribute.get("type") != "vulnerability": + misperrors["error"] = "Vulnerability id missing." return misperrors - api_url = check_url(request['config']['custom_API']) if request.get('config', {}).get('custom_API') else cveapi_url + api_url = check_url(request["config"]["custom_API"]) if request.get("config", {}).get("custom_API") else cveapi_url r = requests.get(f"{api_url}cve/{attribute['value']}") if r.status_code == 200: vulnerability = r.json() if not vulnerability: - misperrors['error'] = 'Non existing CVE' - return misperrors['error'] + misperrors["error"] = "Non existing CVE" + return misperrors["error"] else: - misperrors['error'] = 'API not accessible' - return misperrors['error'] + misperrors["error"] = "API not accessible" + return misperrors["error"] parser = VulnerabilityParser(attribute, api_url) parser.parse_vulnerability_information(vulnerability) return parser.get_result() @@ -181,5 +201,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/cytomic_orion.py b/misp_modules/modules/expansion/cytomic_orion.py index 41750bd9d..1f11c0150 100755 --- a/misp_modules/modules/expansion/cytomic_orion.py +++ b/misp_modules/modules/expansion/cytomic_orion.py @@ -1,39 +1,62 @@ #!/usr/bin/env python3 -''' +""" Cytomic Orion MISP Module An expansion module to enrich attributes in MISP and share indicators of compromise with Cytomic Orion -''' +""" -from . import check_input_attribute, standard_error_message -from pymisp import MISPAttribute, MISPEvent, MISPObject import json -import requests import sys -misperrors = {'error': 'Error'} -mispattributes = {'input': ['md5'], 'format': 'misp_standard'} +import requests +from pymisp import MISPAttribute, MISPEvent, MISPObject + +from . import check_input_attribute, standard_error_message + +misperrors = {"error": "Error"} +mispattributes = {"input": ["md5"], "format": "misp_standard"} moduleinfo = { - 'version': '0.3', - 'author': 'Koen Van Impe', - 'description': 'An expansion module to enrich attributes in MISP by quering the Cytomic Orion API', - 'module-type': ['expansion'], - 'name': 'Cytomic Orion Lookup', - 'logo': 'cytomic_orion.png', - 'requirements': ['Access (license) to Cytomic Orion'], - 'features': 'This module takes an MD5 hash and searches for occurrences of this hash in the Cytomic Orion database. Returns observed files and machines.', - 'references': ['https://www.vanimpe.eu/2020/03/10/integrating-misp-and-cytomic-orion/', 'https://www.cytomicmodel.com/solutions/'], - 'input': 'MD5, hash of the sample / malware to search for.', - 'output': 'MISP objects with sightings of the hash in Cytomic Orion. Includes files and machines.', + "version": "0.3", + "author": "Koen Van Impe", + "description": "An expansion module to enrich attributes in MISP by quering the Cytomic Orion API", + "module-type": ["expansion"], + "name": "Cytomic Orion Lookup", + "logo": "cytomic_orion.png", + "requirements": ["Access (license) to Cytomic Orion"], + "features": ( + "This module takes an MD5 hash and searches for occurrences of this hash in the Cytomic Orion database. Returns" + " observed files and machines." + ), + "references": [ + "https://www.vanimpe.eu/2020/03/10/integrating-misp-and-cytomic-orion/", + "https://www.cytomicmodel.com/solutions/", + ], + "input": "MD5, hash of the sample / malware to search for.", + "output": "MISP objects with sightings of the hash in Cytomic Orion. Includes files and machines.", } -moduleconfig = ['api_url', 'token_url', 'clientid', 'clientsecret', 'clientsecret', 'username', 'password', 'upload_timeframe', 'upload_tag', 'delete_tag', 'upload_ttlDays', 'upload_threat_level_id', 'limit_upload_events', 'limit_upload_attributes'] +moduleconfig = [ + "api_url", + "token_url", + "clientid", + "clientsecret", + "clientsecret", + "username", + "password", + "upload_timeframe", + "upload_tag", + "delete_tag", + "upload_ttlDays", + "upload_threat_level_id", + "limit_upload_events", + "limit_upload_attributes", +] # There are more config settings in this module than used by the enrichment # There is also a PyMISP module which reuses the module config, and requires additional configuration, for example used for pushing indicators to the API -class CytomicParser(): +class CytomicParser: def __init__(self, attribute, config_object): self.misp_event = MISPEvent() self.attribute = MISPAttribute() @@ -45,107 +68,188 @@ def __init__(self, attribute, config_object): if self.config_object: self.token = self.get_token() else: - sys.exit('Missing configuration') + sys.exit("Missing configuration") def get_token(self): try: - scope = self.config_object['scope'] - grant_type = self.config_object['grant_type'] - username = self.config_object['username'] - password = self.config_object['password'] - token_url = self.config_object['token_url'] - clientid = self.config_object['clientid'] - clientsecret = self.config_object['clientsecret'] + scope = self.config_object["scope"] + grant_type = self.config_object["grant_type"] + username = self.config_object["username"] + password = self.config_object["password"] + token_url = self.config_object["token_url"] + clientid = self.config_object["clientid"] + clientsecret = self.config_object["clientsecret"] if scope and grant_type and username and password: - data = {'scope': scope, 'grant_type': grant_type, 'username': username, 'password': password} + data = { + "scope": scope, + "grant_type": grant_type, + "username": username, + "password": password, + } if token_url and clientid and clientsecret: - access_token_response = requests.post(token_url, data=data, verify=False, allow_redirects=False, auth=(clientid, clientsecret)) + access_token_response = requests.post( + token_url, + data=data, + verify=False, + allow_redirects=False, + auth=(clientid, clientsecret), + ) tokens = json.loads(access_token_response.text) - if 'access_token' in tokens: - return tokens['access_token'] + if "access_token" in tokens: + return tokens["access_token"] else: - self.result = {'error': 'No token received.'} + self.result = {"error": "No token received."} return else: - self.result = {'error': 'No token_url, clientid or clientsecret supplied.'} + self.result = {"error": "No token_url, clientid or clientsecret supplied."} return else: - self.result = {'error': 'No scope, grant_type, username or password supplied.'} + self.result = {"error": "No scope, grant_type, username or password supplied."} return except Exception: - self.result = {'error': 'Unable to connect to token_url.'} + self.result = {"error": "Unable to connect to token_url."} return def get_results(self): - if hasattr(self, 'result'): + if hasattr(self, "result"): return self.result event = json.loads(self.misp_event.to_json()) - results = {key: event[key] for key in ('Attribute', 'Object')} - return {'results': results} + results = {key: event[key] for key in ("Attribute", "Object")} + return {"results": results} def parse(self, searchkey): if self.token: - endpoint_fileinformation = self.config_object['endpoint_fileinformation'] - endpoint_machines = self.config_object['endpoint_machines'] - endpoint_machines_client = self.config_object['endpoint_machines_client'] - query_machines = self.config_object['query_machines'] - query_machine_info = self.config_object['query_machine_info'] + endpoint_fileinformation = self.config_object["endpoint_fileinformation"] + endpoint_machines = self.config_object["endpoint_machines"] + endpoint_machines_client = self.config_object["endpoint_machines_client"] + query_machines = self.config_object["query_machines"] + query_machine_info = self.config_object["query_machine_info"] # Update endpoint URLs query_endpoint_fileinformation = endpoint_fileinformation.format(md5=searchkey) query_endpoint_machines = endpoint_machines.format(md5=searchkey) # API calls - api_call_headers = {'Authorization': 'Bearer ' + self.token} - result_query_endpoint_fileinformation = requests.get(query_endpoint_fileinformation, headers=api_call_headers, verify=False) + api_call_headers = {"Authorization": "Bearer " + self.token} + result_query_endpoint_fileinformation = requests.get( + query_endpoint_fileinformation, headers=api_call_headers, verify=False + ) json_result_query_endpoint_fileinformation = json.loads(result_query_endpoint_fileinformation.text) if json_result_query_endpoint_fileinformation: - cytomic_object = MISPObject('cytomic-orion-file') - - cytomic_object.add_attribute('fileName', type='text', value=json_result_query_endpoint_fileinformation['fileName']) - cytomic_object.add_attribute('fileSize', type='text', value=json_result_query_endpoint_fileinformation['fileSize']) - cytomic_object.add_attribute('last-seen', type='datetime', value=json_result_query_endpoint_fileinformation['lastSeen']) - cytomic_object.add_attribute('first-seen', type='datetime', value=json_result_query_endpoint_fileinformation['firstSeen']) - cytomic_object.add_attribute('classification', type='text', value=json_result_query_endpoint_fileinformation['classification']) - cytomic_object.add_attribute('classificationName', type='text', value=json_result_query_endpoint_fileinformation['classificationName']) + cytomic_object = MISPObject("cytomic-orion-file") + + cytomic_object.add_attribute( + "fileName", + type="text", + value=json_result_query_endpoint_fileinformation["fileName"], + ) + cytomic_object.add_attribute( + "fileSize", + type="text", + value=json_result_query_endpoint_fileinformation["fileSize"], + ) + cytomic_object.add_attribute( + "last-seen", + type="datetime", + value=json_result_query_endpoint_fileinformation["lastSeen"], + ) + cytomic_object.add_attribute( + "first-seen", + type="datetime", + value=json_result_query_endpoint_fileinformation["firstSeen"], + ) + cytomic_object.add_attribute( + "classification", + type="text", + value=json_result_query_endpoint_fileinformation["classification"], + ) + cytomic_object.add_attribute( + "classificationName", + type="text", + value=json_result_query_endpoint_fileinformation["classificationName"], + ) self.misp_event.add_object(**cytomic_object) - result_query_endpoint_machines = requests.get(query_endpoint_machines, headers=api_call_headers, verify=False) + result_query_endpoint_machines = requests.get( + query_endpoint_machines, headers=api_call_headers, verify=False + ) json_result_query_endpoint_machines = json.loads(result_query_endpoint_machines.text) - if query_machines and json_result_query_endpoint_machines and len(json_result_query_endpoint_machines) > 0: + if ( + query_machines + and json_result_query_endpoint_machines + and len(json_result_query_endpoint_machines) > 0 + ): for machine in json_result_query_endpoint_machines: - if query_machine_info and machine['muid']: - query_endpoint_machines_client = endpoint_machines_client.format(muid=machine['muid']) - result_endpoint_machines_client = requests.get(query_endpoint_machines_client, headers=api_call_headers, verify=False) + if query_machine_info and machine["muid"]: + query_endpoint_machines_client = endpoint_machines_client.format(muid=machine["muid"]) + result_endpoint_machines_client = requests.get( + query_endpoint_machines_client, + headers=api_call_headers, + verify=False, + ) json_result_endpoint_machines_client = json.loads(result_endpoint_machines_client.text) if json_result_endpoint_machines_client: - cytomic_machine_object = MISPObject('cytomic-orion-machine') - - clienttag = [{'name': json_result_endpoint_machines_client['clientName']}] - - cytomic_machine_object.add_attribute('machineName', type='target-machine', value=json_result_endpoint_machines_client['machineName'], Tag=clienttag) - cytomic_machine_object.add_attribute('machineMuid', type='text', value=machine['muid']) - cytomic_machine_object.add_attribute('clientName', type='target-org', value=json_result_endpoint_machines_client['clientName'], Tag=clienttag) - cytomic_machine_object.add_attribute('clientId', type='text', value=machine['clientId']) - cytomic_machine_object.add_attribute('machinePath', type='text', value=machine['lastPath']) - cytomic_machine_object.add_attribute('first-seen', type='datetime', value=machine['firstSeen']) - cytomic_machine_object.add_attribute('last-seen', type='datetime', value=machine['lastSeen']) - cytomic_machine_object.add_attribute('creationDate', type='datetime', value=json_result_endpoint_machines_client['creationDate']) - cytomic_machine_object.add_attribute('clientCreationDateUTC', type='datetime', value=json_result_endpoint_machines_client['clientCreationDateUTC']) - cytomic_machine_object.add_attribute('lastSeenUtc', type='datetime', value=json_result_endpoint_machines_client['lastSeenUtc']) + cytomic_machine_object = MISPObject("cytomic-orion-machine") + + clienttag = [{"name": json_result_endpoint_machines_client["clientName"]}] + + cytomic_machine_object.add_attribute( + "machineName", + type="target-machine", + value=json_result_endpoint_machines_client["machineName"], + Tag=clienttag, + ) + cytomic_machine_object.add_attribute("machineMuid", type="text", value=machine["muid"]) + cytomic_machine_object.add_attribute( + "clientName", + type="target-org", + value=json_result_endpoint_machines_client["clientName"], + Tag=clienttag, + ) + cytomic_machine_object.add_attribute("clientId", type="text", value=machine["clientId"]) + cytomic_machine_object.add_attribute( + "machinePath", + type="text", + value=machine["lastPath"], + ) + cytomic_machine_object.add_attribute( + "first-seen", + type="datetime", + value=machine["firstSeen"], + ) + cytomic_machine_object.add_attribute( + "last-seen", + type="datetime", + value=machine["lastSeen"], + ) + cytomic_machine_object.add_attribute( + "creationDate", + type="datetime", + value=json_result_endpoint_machines_client["creationDate"], + ) + cytomic_machine_object.add_attribute( + "clientCreationDateUTC", + type="datetime", + value=json_result_endpoint_machines_client["clientCreationDateUTC"], + ) + cytomic_machine_object.add_attribute( + "lastSeenUtc", + type="datetime", + value=json_result_endpoint_machines_client["lastSeenUtc"], + ) self.misp_event.add_object(**cytomic_machine_object) else: - self.result = {'error': 'No (valid) token.'} + self.result = {"error": "No (valid) token."} return @@ -154,35 +258,44 @@ def handler(q=False): return False request = json.loads(q) - if not request.get('attribute'): - return {'error': 'Unsupported input.'} + if not request.get("attribute"): + return {"error": "Unsupported input."} - if not request.get('attribute') or not check_input_attribute(request['attribute']): - return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} - attribute = request['attribute'] - if not any(input_type == attribute['type'] for input_type in mispattributes['input']): - return {'error': 'Unsupported attribute type.'} + if not request.get("attribute") or not check_input_attribute(request["attribute"]): + return {"error": f"{standard_error_message}, which should contain at least a type, a value and an uuid."} + attribute = request["attribute"] + if not any(input_type == attribute["type"] for input_type in mispattributes["input"]): + return {"error": "Unsupported attribute type."} - if not request.get('config'): - return {'error': 'Missing configuration'} + if not request.get("config"): + return {"error": "Missing configuration"} config_object = { - 'clientid': request["config"].get("clientid"), - 'clientsecret': request["config"].get("clientsecret"), - 'scope': 'orion.api', - 'password': request["config"].get("password"), - 'username': request["config"].get("username"), - 'grant_type': 'password', - 'token_url': request["config"].get("token_url"), - 'endpoint_fileinformation': '{api_url}{endpoint}'.format(api_url=request["config"].get("api_url"), endpoint='/forensics/md5/{md5}/info'), - 'endpoint_machines': '{api_url}{endpoint}'.format(api_url=request["config"].get("api_url"), endpoint='/forensics/md5/{md5}/muids'), - 'endpoint_machines_client': '{api_url}{endpoint}'.format(api_url=request["config"].get("api_url"), endpoint='/forensics/muid/{muid}/info'), - 'query_machines': True, - 'query_machine_info': True + "clientid": request["config"].get("clientid"), + "clientsecret": request["config"].get("clientsecret"), + "scope": "orion.api", + "password": request["config"].get("password"), + "username": request["config"].get("username"), + "grant_type": "password", + "token_url": request["config"].get("token_url"), + "endpoint_fileinformation": "{api_url}{endpoint}".format( + api_url=request["config"].get("api_url"), + endpoint="/forensics/md5/{md5}/info", + ), + "endpoint_machines": "{api_url}{endpoint}".format( + api_url=request["config"].get("api_url"), + endpoint="/forensics/md5/{md5}/muids", + ), + "endpoint_machines_client": "{api_url}{endpoint}".format( + api_url=request["config"].get("api_url"), + endpoint="/forensics/muid/{muid}/info", + ), + "query_machines": True, + "query_machine_info": True, } cytomic_parser = CytomicParser(attribute, config_object) - cytomic_parser.parse(attribute['value']) + cytomic_parser.parse(attribute["value"]) return cytomic_parser.get_results() @@ -192,5 +305,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/dbl_spamhaus.py b/misp_modules/modules/expansion/dbl_spamhaus.py index f27a3ff8c..2244a23cd 100644 --- a/misp_modules/modules/expansion/dbl_spamhaus.py +++ b/misp_modules/modules/expansion/dbl_spamhaus.py @@ -1,52 +1,61 @@ import json import sys -try: - original_path = sys.path - sys.path = original_path[1:] - import dns.resolver - sys.path = original_path - resolver = dns.resolver.Resolver() - resolver.timeout = 0.2 - resolver.lifetime = 0.2 -except ImportError: - print("dnspython3 is missing, use 'pip install dnspython3' to install it.") - sys.exit(0) +original_path = sys.path +sys.path = original_path[1:] +import dns.resolver -misperrors = {'error': 'Error'} -mispattributes = {'input': ['domain', 'domain|ip', 'hostname', 'hostname|port'], 'output': ['text']} +sys.path = original_path +resolver = dns.resolver.Resolver() +resolver.timeout = 0.2 +resolver.lifetime = 0.2 + + +misperrors = {"error": "Error"} +mispattributes = { + "input": ["domain", "domain|ip", "hostname", "hostname|port"], + "output": ["text"], +} moduleinfo = { - 'version': '0.1', - 'author': 'Christian Studer', - 'description': 'Checks Spamhaus DBL for a domain name.', - 'module-type': ['expansion', 'hover'], - 'name': 'DBL Spamhaus Lookup', - 'logo': 'spamhaus.jpg', - 'requirements': ['dnspython3: DNS python3 library'], - 'features': 'This modules takes a domain or a hostname in input and queries the Domain Block List provided by Spamhaus to determine what kind of domain it is.\n\nDBL then returns a response code corresponding to a certain classification of the domain we display. If the queried domain is not in the list, it is also mentionned.\n\nPlease note that composite MISP attributes containing domain or hostname are supported as well.', - 'references': ['https://www.spamhaus.org/faq/section/Spamhaus%20DBL'], - 'input': 'Domain or hostname attribute.', - 'output': 'Information about the nature of the input.', + "version": "0.1", + "author": "Christian Studer", + "description": "Checks Spamhaus DBL for a domain name.", + "module-type": ["expansion", "hover"], + "name": "DBL Spamhaus Lookup", + "logo": "spamhaus.jpg", + "requirements": ["dnspython3: DNS python3 library"], + "features": ( + "This modules takes a domain or a hostname in input and queries the Domain Block List provided by Spamhaus to" + " determine what kind of domain it is.\n\nDBL then returns a response code corresponding to a certain" + " classification of the domain we display. If the queried domain is not in the list, it is also" + " mentionned.\n\nPlease note that composite MISP attributes containing domain or hostname are supported as" + " well." + ), + "references": ["https://www.spamhaus.org/faq/section/Spamhaus%20DBL"], + "input": "Domain or hostname attribute.", + "output": "Information about the nature of the input.", } moduleconfig = [] -dbl = 'dbl.spamhaus.org' -dbl_mapping = {'127.0.1.2': 'spam domain', - '127.0.1.4': 'phish domain', - '127.0.1.5': 'malware domain', - '127.0.1.6': 'botnet C&C domain', - '127.0.1.102': 'abused legit spam', - '127.0.1.103': 'abused spammed redirector domain', - '127.0.1.104': 'abused legit phish', - '127.0.1.105': 'abused legit malware', - '127.0.1.106': 'abused legit botnet C&C', - '127.0.1.255': 'IP queries prohibited!'} +dbl = "dbl.spamhaus.org" +dbl_mapping = { + "127.0.1.2": "spam domain", + "127.0.1.4": "phish domain", + "127.0.1.5": "malware domain", + "127.0.1.6": "botnet C&C domain", + "127.0.1.102": "abused legit spam", + "127.0.1.103": "abused spammed redirector domain", + "127.0.1.104": "abused legit phish", + "127.0.1.105": "abused legit malware", + "127.0.1.106": "abused legit botnet C&C", + "127.0.1.255": "IP queries prohibited!", +} def fetch_requested_value(request): - for attribute_type in mispattributes['input']: + for attribute_type in mispattributes["input"]: if request.get(attribute_type): - return request[attribute_type].split('|')[0] + return request[attribute_type].split("|")[0] return None @@ -56,17 +65,17 @@ def handler(q=False): request = json.loads(q) requested_value = fetch_requested_value(request) if requested_value is None: - misperrors['error'] = "Unsupported attributes type" + misperrors["error"] = "Unsupported attributes type" return misperrors query = "{}.{}".format(requested_value, dbl) try: - query_result = resolver.resolve(query, 'A')[0] + query_result = resolver.resolve(query, "A")[0] result = "{} - {}".format(requested_value, dbl_mapping[str(query_result)]) except dns.resolver.NXDOMAIN as e: result = e.msg except Exception: - return {'error': 'Not able to reach dbl.spamhaus.org or something went wrong'} - return {'results': [{'types': mispattributes.get('output'), 'values': result}]} + return {"error": "Not able to reach dbl.spamhaus.org or something went wrong"} + return {"results": [{"types": mispattributes.get("output"), "values": result}]} def introspection(): @@ -74,5 +83,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/dns.py b/misp_modules/modules/expansion/dns.py index 8654af810..2900d16a1 100755 --- a/misp_modules/modules/expansion/dns.py +++ b/misp_modules/modules/expansion/dns.py @@ -1,64 +1,72 @@ import json + import dns.resolver -misperrors = {'error': 'Error'} -mispattributes = {'input': ['hostname', 'domain', 'domain|ip'], 'output': ['ip-src', - 'ip-dst']} +misperrors = {"error": "Error"} +mispattributes = { + "input": ["hostname", "domain", "domain|ip"], + "output": ["ip-src", "ip-dst"], +} moduleinfo = { - 'version': '0.3', - 'author': 'Alexandre Dulaunoy', - 'description': 'Simple DNS expansion service to resolve IP address from MISP attributes', - 'module-type': ['expansion', 'hover'], - 'name': 'DNS Resolver', - 'logo': '', - 'requirements': ['dnspython3: DNS python3 library'], - 'features': 'The module takes a domain of hostname attribute as input, and tries to resolve it. If no error is encountered, the IP address that resolves the domain is returned, otherwise the origin of the error is displayed.\n\nThe address of the DNS resolver to use is also configurable, but if no configuration is set, we use the Google public DNS address (8.8.8.8).\n\nPlease note that composite MISP attributes containing domain or hostname are supported as well.', - 'references': [], - 'input': 'Domain or hostname attribute.', - 'output': 'IP address resolving the input.', + "version": "0.3", + "author": "Alexandre Dulaunoy", + "description": "Simple DNS expansion service to resolve IP address from MISP attributes", + "module-type": ["expansion", "hover"], + "name": "DNS Resolver", + "logo": "", + "requirements": ["dnspython3: DNS python3 library"], + "features": ( + "The module takes a domain of hostname attribute as input, and tries to resolve it. If no error is encountered," + " the IP address that resolves the domain is returned, otherwise the origin of the error is displayed.\n\nThe" + " address of the DNS resolver to use is also configurable, but if no configuration is set, we use the Google" + " public DNS address (8.8.8.8).\n\nPlease note that composite MISP attributes containing domain or hostname are" + " supported as well." + ), + "references": [], + "input": "Domain or hostname attribute.", + "output": "IP address resolving the input.", } -moduleconfig = ['nameserver'] +moduleconfig = ["nameserver"] def handler(q=False): if q is False: return False request = json.loads(q) - if request.get('hostname'): - toquery = request['hostname'] - elif request.get('domain'): - toquery = request['domain'] - elif request.get('domain|ip'): - toquery = request['domain|ip'].split('|')[0] + if request.get("hostname"): + toquery = request["hostname"] + elif request.get("domain"): + toquery = request["domain"] + elif request.get("domain|ip"): + toquery = request["domain|ip"].split("|")[0] else: return False r = dns.resolver.Resolver() r.timeout = 2 r.lifetime = 2 - if request.get('config'): - if request['config'].get('nameserver'): + if request.get("config"): + if request["config"].get("nameserver"): nameservers = [] - nameservers.append(request['config'].get('nameserver')) + nameservers.append(request["config"].get("nameserver")) r.nameservers = nameservers else: - r.nameservers = ['8.8.8.8'] + r.nameservers = ["8.8.8.8"] try: - answer = r.resolve(toquery, 'A') + answer = r.resolve(toquery, "A") except dns.resolver.NXDOMAIN: - misperrors['error'] = "NXDOMAIN" + misperrors["error"] = "NXDOMAIN" return misperrors except dns.exception.Timeout: - misperrors['error'] = "Timeout" + misperrors["error"] = "Timeout" return misperrors except Exception as e: - misperrors['error'] = f'DNS resolving error {e}' + misperrors["error"] = f"DNS resolving error {e}" return misperrors - r = {'results': [{'types': mispattributes['output'], - 'values':[str(answer[0])]}]} + r = {"results": [{"types": mispattributes["output"], "values": [str(answer[0])]}]} return r @@ -67,5 +75,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/docx_enrich.py b/misp_modules/modules/expansion/docx_enrich.py index aaf269df8..78ee07432 100644 --- a/misp_modules/modules/expansion/docx_enrich.py +++ b/misp_modules/modules/expansion/docx_enrich.py @@ -1,24 +1,27 @@ -import json import binascii -import np -import docx import io +import json + +import docx +import np -misperrors = {'error': 'Error'} -mispattributes = {'input': ['attachment'], - 'output': ['freetext', 'text']} +misperrors = {"error": "Error"} +mispattributes = {"input": ["attachment"], "output": ["freetext", "text"]} moduleinfo = { - 'version': '0.1', - 'author': 'Sascha Rommelfangen', - 'description': 'Module to extract freetext from a .docx document.', - 'module-type': ['expansion'], - 'name': 'DOCX Enrich', - 'logo': 'docx.png', - 'requirements': ['docx python library'], - 'features': 'The module reads the text contained in a .docx document. The result is passed to the freetext import parser so IoCs can be extracted out of it.', - 'references': [], - 'input': 'Attachment attribute containing a .docx document.', - 'output': 'Text and freetext parsed from the document.', + "version": "0.1", + "author": "Sascha Rommelfangen", + "description": "Module to extract freetext from a .docx document.", + "module-type": ["expansion"], + "name": "DOCX Enrich", + "logo": "docx.png", + "requirements": ["docx python library"], + "features": ( + "The module reads the text contained in a .docx document. The result is passed to the freetext import parser so" + " IoCs can be extracted out of it." + ), + "references": [], + "input": "Attachment attribute containing a .docx document.", + "output": "Text and freetext parsed from the document.", } moduleconfig = [] @@ -28,13 +31,13 @@ def handler(q=False): if q is False: return False q = json.loads(q) - filename = q['attachment'] + filename = q["attachment"] try: - docx_array = np.frombuffer(binascii.a2b_base64(q['data']), np.uint8) + docx_array = np.frombuffer(binascii.a2b_base64(q["data"]), np.uint8) except Exception as e: print(e) err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?" - misperrors['error'] = err + misperrors["error"] = err print(err) return misperrors @@ -53,12 +56,24 @@ def handler(q=False): print(para.text) doc_content = doc_content + "\n" + para.text print(doc_content) - return {'results': [{'types': ['freetext'], 'values': doc_content, 'comment': ".docx-to-text from file " + filename}, - {'types': ['text'], 'values': doc_content, 'comment': ".docx-to-text from file " + filename}]} + return { + "results": [ + { + "types": ["freetext"], + "values": doc_content, + "comment": ".docx-to-text from file " + filename, + }, + { + "types": ["text"], + "values": doc_content, + "comment": ".docx-to-text from file " + filename, + }, + ] + } except Exception as e: print(e) err = "Couldn't analyze file as .docx. Error was: " + str(e) - misperrors['error'] = err + misperrors["error"] = err return misperrors @@ -67,5 +82,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/domaintools.py b/misp_modules/modules/expansion/domaintools.py index c70b7f7ec..d270ce9a4 100755 --- a/misp_modules/modules/expansion/domaintools.py +++ b/misp_modules/modules/expansion/domaintools.py @@ -8,43 +8,86 @@ from domaintools import API - -log = logging.getLogger('domaintools') +log = logging.getLogger("domaintools") log.setLevel(logging.DEBUG) ch = logging.StreamHandler(sys.stdout) ch.setLevel(logging.DEBUG) -formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") ch.setFormatter(formatter) log.addHandler(ch) -misperrors = {'error': 'Error'} +misperrors = {"error": "Error"} mispattributes = { - 'input': ['domain', 'email-src', 'email-dst', 'target-email', 'whois-registrant-email', - 'whois-registrant-name', 'whois-registrant-phone', 'ip-src', 'ip-dst'], - 'output': ['whois-registrant-email', 'whois-registrant-phone', 'whois-registrant-name', - 'whois-registrar', 'whois-creation-date', 'freetext', 'domain'] + "input": [ + "domain", + "email-src", + "email-dst", + "target-email", + "whois-registrant-email", + "whois-registrant-name", + "whois-registrant-phone", + "ip-src", + "ip-dst", + ], + "output": [ + "whois-registrant-email", + "whois-registrant-phone", + "whois-registrant-name", + "whois-registrar", + "whois-creation-date", + "freetext", + "domain", + ], } moduleinfo = { - 'version': '0.1', - 'author': 'Raphaël Vinot', - 'description': 'DomainTools MISP expansion module.', - 'module-type': ['expansion', 'hover'], - 'name': 'DomainTools Lookup', - 'logo': 'domaintools.png', - 'requirements': ['Domaintools python library', 'A Domaintools API access (username & apikey)'], - 'features': 'This module takes a MISP attribute as input to query the Domaintools API. The API returns then the result of the query with some types we map into compatible types we add as MISP attributes.\n\nPlease note that composite attributes composed by at least one of the input types mentionned below (domains, IPs, hostnames) are also supported.', - 'references': ['https://www.domaintools.com/'], - 'input': 'A MISP attribute included in the following list:\n- domain\n- hostname\n- email-src\n- email-dst\n- target-email\n- whois-registrant-email\n- whois-registrant-name\n- whois-registrant-phone\n- ip-src\n- ip-dst', - 'output': 'MISP attributes mapped after the Domaintools API has been queried, included in the following list:\n- whois-registrant-email\n- whois-registrant-phone\n- whois-registrant-name\n- whois-registrar\n- whois-creation-date\n- text\n- domain', + "version": "0.1", + "author": "Raphaël Vinot", + "description": "DomainTools MISP expansion module.", + "module-type": ["expansion", "hover"], + "name": "DomainTools Lookup", + "logo": "domaintools.png", + "requirements": [ + "Domaintools python library", + "A Domaintools API access (username & apikey)", + ], + "features": ( + "This module takes a MISP attribute as input to query the Domaintools API. The API returns then the result of" + " the query with some types we map into compatible types we add as MISP attributes.\n\nPlease note that" + " composite attributes composed by at least one of the input types mentionned below (domains, IPs, hostnames)" + " are also supported." + ), + "references": ["https://www.domaintools.com/"], + "input": ( + "A MISP attribute included in the following list:\n- domain\n- hostname\n- email-src\n- email-dst\n-" + " target-email\n- whois-registrant-email\n- whois-registrant-name\n- whois-registrant-phone\n- ip-src\n- ip-dst" + ), + "output": ( + "MISP attributes mapped after the Domaintools API has been queried, included in the following list:\n-" + " whois-registrant-email\n- whois-registrant-phone\n- whois-registrant-name\n- whois-registrar\n-" + " whois-creation-date\n- text\n- domain" + ), } -moduleconfig = ['username', 'api_key'] +moduleconfig = ["username", "api_key"] query_profiles = [ - {'inputs': ['domain'], 'services': ['parsed_whois', 'domain_profile', 'reputation', 'reverse_ip']}, - {'inputs': ['email-src', 'email-dst', 'target-email', 'whois-registrant-email', 'whois-registrant-name', 'whois-registrant-phone'], 'services': ['reverse_whois']}, - {'inputs': ['ip-src', 'ip-dst'], 'services': ['host_domains']} + { + "inputs": ["domain"], + "services": ["parsed_whois", "domain_profile", "reputation", "reverse_ip"], + }, + { + "inputs": [ + "email-src", + "email-dst", + "target-email", + "whois-registrant-email", + "whois-registrant-name", + "whois-registrant-phone", + ], + "services": ["reverse_whois"], + }, + {"inputs": ["ip-src", "ip-dst"], "services": ["host_domains"]}, ] @@ -59,14 +102,14 @@ def __init__(self): self.domain_ip = {} self.domain = {} self.risk = () - self.freetext = '' + self.freetext = "" def _add_value(self, value_type, value, comment): if value_type.get(value): if comment and comment not in value_type[value]: - value_type[value] += ' - {}'.format(comment) + value_type[value] += " - {}".format(comment) else: - value_type[value] = comment or '' + value_type[value] = comment or "" return value_type def add_mail(self, mail, comment=None): @@ -94,132 +137,186 @@ def dump(self): to_return = [] if self.reg_mail: for mail, comment in self.reg_mail.items(): - to_return.append({'type': 'whois-registrant-email', 'values': [mail], 'comment': comment or ''}) + to_return.append( + { + "type": "whois-registrant-email", + "values": [mail], + "comment": comment or "", + } + ) if self.reg_phone: for phone, comment in self.reg_phone.items(): - to_return.append({'type': 'whois-registrant-phone', 'values': [phone], 'comment': comment or ''}) + to_return.append( + { + "type": "whois-registrant-phone", + "values": [phone], + "comment": comment or "", + } + ) if self.reg_name: for name, comment in self.reg_name.items(): - to_return.append({'type': 'whois-registrant-name', 'values': [name], 'comment': comment or ''}) + to_return.append( + { + "type": "whois-registrant-name", + "values": [name], + "comment": comment or "", + } + ) if self.registrar: for reg, comment in self.registrar.items(): - to_return.append({'type': 'whois-registrar', 'values': [reg], 'comment': comment or ''}) + to_return.append( + { + "type": "whois-registrar", + "values": [reg], + "comment": comment or "", + } + ) if self.creation_date: for date, comment in self.creation_date.items(): - to_return.append({'type': 'whois-creation-date', 'values': [date], 'comment': comment or ''}) + to_return.append( + { + "type": "whois-creation-date", + "values": [date], + "comment": comment or "", + } + ) if self.domain_ip: for ip, comment in self.domain_ip.items(): - to_return.append({'types': ['ip-dst', 'ip-src'], 'values': [ip], 'comment': comment or ''}) + to_return.append( + { + "types": ["ip-dst", "ip-src"], + "values": [ip], + "comment": comment or "", + } + ) if self.domain: for domain, comment in self.domain.items(): - to_return.append({'type': 'domain', 'values': [domain], 'comment': comment or ''}) + to_return.append({"type": "domain", "values": [domain], "comment": comment or ""}) if self.freetext: - to_return.append({'type': 'freetext', 'values': [self.freetext], 'comment': 'Freetext import'}) + to_return.append( + { + "type": "freetext", + "values": [self.freetext], + "comment": "Freetext import", + } + ) if self.risk: - to_return.append({'type': 'text', 'values': [self.risk[0]], 'comment': self.risk[1]}) + to_return.append({"type": "text", "values": [self.risk[0]], "comment": self.risk[1]}) return to_return def parsed_whois(domtools, to_query, values): whois_entry = domtools.parsed_whois(to_query) - if whois_entry.get('error'): - misperrors['error'] = whois_entry['error']['message'] + if whois_entry.get("error"): + misperrors["error"] = whois_entry["error"]["message"] return misperrors - if whois_entry.get('registrant'): - values.add_name(whois_entry['registrant'], 'Parsed registrant') - - if whois_entry.get('registration'): - values.add_creation_date(whois_entry['registration']['created'], 'timestamp') - - if whois_entry.get('whois'): - values.freetext = whois_entry['whois']['record'] - if whois_entry.get('parsed_whois'): - if whois_entry['parsed_whois']['created_date']: - values.add_creation_date(whois_entry['parsed_whois']['created_date'], 'created') - if whois_entry['parsed_whois']['registrar']['name']: - values.add_registrar(whois_entry['parsed_whois']['registrar']['name'], 'name') - if whois_entry['parsed_whois']['registrar']['url']: - values.add_registrar(whois_entry['parsed_whois']['registrar']['url'], 'url') - if whois_entry['parsed_whois']['registrar']['iana_id']: - values.add_registrar(whois_entry['parsed_whois']['registrar']['iana_id'], 'iana_id') - for key, entry in whois_entry['parsed_whois']['contacts'].items(): - if entry['email']: - values.add_mail(entry['email'], key) - if entry['phone']: - values.add_phone(entry['phone'], key) - if entry['name']: - values.add_name(entry['name'], key) + if whois_entry.get("registrant"): + values.add_name(whois_entry["registrant"], "Parsed registrant") + + if whois_entry.get("registration"): + values.add_creation_date(whois_entry["registration"]["created"], "timestamp") + + if whois_entry.get("whois"): + values.freetext = whois_entry["whois"]["record"] + if whois_entry.get("parsed_whois"): + if whois_entry["parsed_whois"]["created_date"]: + values.add_creation_date(whois_entry["parsed_whois"]["created_date"], "created") + if whois_entry["parsed_whois"]["registrar"]["name"]: + values.add_registrar(whois_entry["parsed_whois"]["registrar"]["name"], "name") + if whois_entry["parsed_whois"]["registrar"]["url"]: + values.add_registrar(whois_entry["parsed_whois"]["registrar"]["url"], "url") + if whois_entry["parsed_whois"]["registrar"]["iana_id"]: + values.add_registrar(whois_entry["parsed_whois"]["registrar"]["iana_id"], "iana_id") + for key, entry in whois_entry["parsed_whois"]["contacts"].items(): + if entry["email"]: + values.add_mail(entry["email"], key) + if entry["phone"]: + values.add_phone(entry["phone"], key) + if entry["name"]: + values.add_name(entry["name"], key) if whois_entry.emails(): for mail in whois_entry.emails(): if mail not in values.reg_mail.keys(): - values.add_mail(mail, 'Maybe registrar') + values.add_mail(mail, "Maybe registrar") return values def domain_profile(domtools, to_query, values): profile = domtools.domain_profile(to_query) # NOTE: profile['website_data']['response_code'] could be used to see if the host is still up. Maybe set a tag. - if profile.get('error'): - misperrors['error'] = profile['error']['message'] + if profile.get("error"): + misperrors["error"] = profile["error"]["message"] return misperrors - if profile.get('registrant'): - values.add_name(profile['registrant']['name'], 'Profile registrant') - - if profile.get('server'): - other_domains = profile['server']['other_domains'] - values.add_ip(profile['server']['ip_address'], 'IP of {} (via DomainTools). Has {} other domains.'.format(to_query, other_domains)) - - if profile.get('registration'): - if profile['registration'].get('created'): - values.add_creation_date(profile['registration']['created'], 'created') - if profile['registration'].get('updated'): - values.add_creation_date(profile['registration']['updated'], 'updated') - if profile['registration'].get('registrar'): - values.add_registrar(profile['registration']['registrar'], 'name') + if profile.get("registrant"): + values.add_name(profile["registrant"]["name"], "Profile registrant") + + if profile.get("server"): + other_domains = profile["server"]["other_domains"] + values.add_ip( + profile["server"]["ip_address"], + "IP of {} (via DomainTools). Has {} other domains.".format(to_query, other_domains), + ) + + if profile.get("registration"): + if profile["registration"].get("created"): + values.add_creation_date(profile["registration"]["created"], "created") + if profile["registration"].get("updated"): + values.add_creation_date(profile["registration"]["updated"], "updated") + if profile["registration"].get("registrar"): + values.add_registrar(profile["registration"]["registrar"], "name") return values def reputation(domtools, to_query, values): rep = domtools.reputation(to_query, include_reasons=True) # NOTE: use that value in a tag when we will have attribute level tagging - if rep and not rep.get('error'): - reasons = ', '.join(rep['reasons']) - values.risk = [rep['risk_score'], 'Risk value of {} (via Domain Tools), Reasons: {}'.format(to_query, reasons)] + if rep and not rep.get("error"): + reasons = ", ".join(rep["reasons"]) + values.risk = [ + rep["risk_score"], + "Risk value of {} (via Domain Tools), Reasons: {}".format(to_query, reasons), + ] return values def reverse_ip(domtools, to_query, values): rev_ip = domtools.reverse_ip(to_query) - if rev_ip and not rev_ip.get('error'): - ip_addresses = rev_ip['ip_addresses'] - values.add_ip(ip_addresses['ip_address'], 'IP of {} (via DomainTools). Has {} other domains.'.format(to_query, ip_addresses['domain_count'])) - for d in ip_addresses['domain_names']: - values.add_domain(d, 'Other domain on {}.'.format(ip_addresses['ip_address'])) + if rev_ip and not rev_ip.get("error"): + ip_addresses = rev_ip["ip_addresses"] + values.add_ip( + ip_addresses["ip_address"], + "IP of {} (via DomainTools). Has {} other domains.".format(to_query, ip_addresses["domain_count"]), + ) + for d in ip_addresses["domain_names"]: + values.add_domain(d, "Other domain on {}.".format(ip_addresses["ip_address"])) return values def reverse_whois(domtools, to_query, values): - rev_whois = domtools.reverse_whois(to_query, mode='purchase') - if rev_whois.get('error'): - misperrors['error'] = rev_whois['error']['message'] + rev_whois = domtools.reverse_whois(to_query, mode="purchase") + if rev_whois.get("error"): + misperrors["error"] = rev_whois["error"]["message"] return misperrors - for d in rev_whois['domains']: - values.add_domain(d, 'Reverse domain related to {}.'.format(to_query)) + for d in rev_whois["domains"]: + values.add_domain(d, "Reverse domain related to {}.".format(to_query)) return values def host_domains(domtools, to_query, values): hostdom = domtools.host_domains(to_query) - if hostdom.get('error'): - misperrors['error'] = hostdom['error']['message'] + if hostdom.get("error"): + misperrors["error"] = hostdom["error"]["message"] return misperrors - ip_addresses = hostdom['ip_addresses'] - if to_query != ip_addresses['ip_address']: - values.add_ip(ip_addresses['ip_address'], 'IP of {} (via DomainTools). Has {} other domains.'.format(to_query, ip_addresses['domain_count'])) - for d in ip_addresses['domain_names']: - values.add_domain(d, 'Other domain on {}.'.format(ip_addresses['ip_address'])) + ip_addresses = hostdom["ip_addresses"] + if to_query != ip_addresses["ip_address"]: + values.add_ip( + ip_addresses["ip_address"], + "IP of {} (via DomainTools). Has {} other domains.".format(to_query, ip_addresses["domain_count"]), + ) + for d in ip_addresses["domain_names"]: + values.add_domain(d, "Other domain on {}.".format(ip_addresses["ip_address"])) return values @@ -227,8 +324,8 @@ def reverse_ip_whois(domtools, to_query, values): # Disabled for now, dies with domaintools.exceptions.NotAuthorizedException rev_whois = domtools.reverse_ip_whois(ip=to_query) print(rev_whois) - if rev_whois.get('error'): - misperrors['error'] = rev_whois['error']['message'] + if rev_whois.get("error"): + misperrors["error"] = rev_whois["error"]["message"] return misperrors # for d in rev_whois['domains']: # values.add_domain(d, 'Reverse domain related to {}.'.format(to_query)) @@ -236,13 +333,13 @@ def reverse_ip_whois(domtools, to_query, values): def get_services(request): - for t in mispattributes['input']: + for t in mispattributes["input"]: to_query = request.get(t) if not to_query: continue for p in query_profiles: - if t in p['inputs']: - return p['services'] + if t in p["inputs"]: + return p["services"] def handler(q=False): @@ -251,22 +348,22 @@ def handler(q=False): request = json.loads(q) to_query = None - for t in mispattributes['input']: + for t in mispattributes["input"]: to_query = request.get(t) if to_query: break if not to_query: - misperrors['error'] = "Unsupported attributes type" + misperrors["error"] = "Unsupported attributes type" return misperrors - if request.get('config'): - if (request['config'].get('username') is None) or (request['config'].get('api_key') is None): - misperrors['error'] = 'DomainTools authentication is incomplete' + if request.get("config"): + if (request["config"].get("username") is None) or (request["config"].get("api_key") is None): + misperrors["error"] = "DomainTools authentication is incomplete" return misperrors else: - domtools = API(request['config'].get('username'), request['config'].get('api_key')) + domtools = API(request["config"].get("username"), request["config"].get("api_key")) else: - misperrors['error'] = 'DomainTools authentication is missing' + misperrors["error"] = "DomainTools authentication is missing" return misperrors values = DomainTools() @@ -278,7 +375,7 @@ def handler(q=False): except Exception as e: print(to_query, type(e), e) - return {'results': values.dump()} + return {"results": values.dump()} def introspection(): @@ -286,5 +383,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/eql.py b/misp_modules/modules/expansion/eql.py index e2b51bd7e..106d85e4b 100644 --- a/misp_modules/modules/expansion/eql.py +++ b/misp_modules/modules/expansion/eql.py @@ -1,43 +1,44 @@ """ Export module for converting MISP events into Endgame EQL queries """ + import json import logging misperrors = {"error": "Error"} moduleinfo = { - 'version': '0.1', - 'author': '92 COS DOM', - 'description': 'EQL query generation for a MISP attribute.', - 'module-type': ['expansion'], - 'name': 'EQL Query Generator', - 'logo': 'eql.png', - 'requirements': [], - 'features': 'This module adds a new attribute to a MISP event containing an EQL query for a network or file attribute.', - 'references': ['https://eql.readthedocs.io/en/latest/'], - 'input': 'A filename or ip attribute.', - 'output': 'Attribute containing EQL for a network or file attribute.', + "version": "0.1", + "author": "92 COS DOM", + "description": "EQL query generation for a MISP attribute.", + "module-type": ["expansion"], + "name": "EQL Query Generator", + "logo": "eql.png", + "requirements": [], + "features": ( + "This module adds a new attribute to a MISP event containing an EQL query for a network or file attribute." + ), + "references": ["https://eql.readthedocs.io/en/latest/"], + "input": "A filename or ip attribute.", + "output": "Attribute containing EQL for a network or file attribute.", } # Map of MISP fields => Endgame fields fieldmap = { "ip-src": "source_address", "ip-dst": "destination_address", - "filename": "file_name" + "filename": "file_name", } # Describe what events have what fields event_types = { "source_address": "network", "destination_address": "network", - "file_name": "file" + "file_name": "file", } # combine all the MISP fields from fieldmap into one big list -mispattributes = { - "input": list(fieldmap.keys()) -} +mispattributes = {"input": list(fieldmap.keys())} def handler(q=False): @@ -61,14 +62,21 @@ def handler(q=False): if attrType: eqlType = fieldmap[attrType] event_type = event_types[eqlType] - fullEql = "{} where {} == \"{}\"".format(event_type, eqlType, request[attrType]) + fullEql = '{} where {} == "{}"'.format(event_type, eqlType, request[attrType]) else: - misperrors['error'] = "Unsupported attributes type" + misperrors["error"] = "Unsupported attributes type" return misperrors response = [] - response.append({'types': ['comment'], 'categories': ['External analysis'], 'values': fullEql, 'comment': "Event EQL queries"}) - return {'results': response} + response.append( + { + "types": ["comment"], + "categories": ["External analysis"], + "values": fullEql, + "comment": "Event EQL queries", + } + ) + return {"results": response} def introspection(): diff --git a/misp_modules/modules/expansion/eupi.py b/misp_modules/modules/expansion/eupi.py index 9b6f94812..0cda39140 100755 --- a/misp_modules/modules/expansion/eupi.py +++ b/misp_modules/modules/expansion/eupi.py @@ -1,48 +1,56 @@ # -*- coding: utf-8 -*- import json + from pyeupi import PyEUPI -misperrors = {'error': 'Error'} -mispattributes = {'input': ['hostname', 'domain', 'url'], 'output': ['freetext']} +misperrors = {"error": "Error"} +mispattributes = {"input": ["hostname", "domain", "url"], "output": ["freetext"]} moduleinfo = { - 'version': '0.1', - 'author': 'Raphaël Vinot', - 'description': 'A module to query the Phishing Initiative service (https://phishing-initiative.lu).', - 'module-type': ['expansion', 'hover'], - 'name': 'EUPI Lookup', - 'logo': 'eupi.png', - 'requirements': ['pyeupi: eupi python library', 'An access to the Phishing Initiative API (apikey & url)'], - 'features': 'This module takes a domain, hostname or url MISP attribute as input to query the Phishing Initiative API. The API returns then the result of the query with some information about the value queried.\n\nPlease note that composite attributes containing domain or hostname are also supported.', - 'references': ['https://phishing-initiative.eu/?lang=en'], - 'input': 'A domain, hostname or url MISP attribute.', - 'output': 'Text containing information about the input, resulting from the query on Phishing Initiative.', + "version": "0.1", + "author": "Raphaël Vinot", + "description": "A module to query the Phishing Initiative service (https://phishing-initiative.lu).", + "module-type": ["expansion", "hover"], + "name": "EUPI Lookup", + "logo": "eupi.png", + "requirements": [ + "pyeupi: eupi python library", + "An access to the Phishing Initiative API (apikey & url)", + ], + "features": ( + "This module takes a domain, hostname or url MISP attribute as input to query the Phishing Initiative API. The" + " API returns then the result of the query with some information about the value queried.\n\nPlease note that" + " composite attributes containing domain or hostname are also supported." + ), + "references": ["https://phishing-initiative.eu/?lang=en"], + "input": "A domain, hostname or url MISP attribute.", + "output": "Text containing information about the input, resulting from the query on Phishing Initiative.", } -moduleconfig = ['apikey', 'url'] +moduleconfig = ["apikey", "url"] def handler(q=False): if q is False: return False request = json.loads(q) - if request.get('hostname'): - toquery = request['hostname'] - elif request.get('domain'): - toquery = request['domain'] - elif request.get('url'): - toquery = request['url'] + if request.get("hostname"): + toquery = request["hostname"] + elif request.get("domain"): + toquery = request["domain"] + elif request.get("url"): + toquery = request["url"] else: - misperrors['error'] = "Unsupported attributes type" + misperrors["error"] = "Unsupported attributes type" return misperrors - if not request.get('config') and not (request['config'].get('apikey') and request['config'].get('url')): - misperrors['error'] = 'EUPI authentication is missing' + if not request.get("config") and not (request["config"].get("apikey") and request["config"].get("url")): + misperrors["error"] = "EUPI authentication is missing" return misperrors - pyeupi = PyEUPI(request['config']['apikey'], request['config']['url']) + pyeupi = PyEUPI(request["config"]["apikey"], request["config"]["url"]) - if 'event_id' in request: + if "event_id" in request: return handle_expansion(pyeupi, toquery) else: return handle_hover(pyeupi, toquery) @@ -51,30 +59,29 @@ def handler(q=False): def handle_expansion(pyeupi, url): results = pyeupi.search_url(url=url) - if results.get('results'): - to_return = '' - for r in results['results']: - if r['tag_label'] != 'phishing': + if results.get("results"): + to_return = "" + for r in results["results"]: + if r["tag_label"] != "phishing": continue - to_return += ' {} {} {} '.format(r['url'], r['domain'], r['ip_address']) + to_return += " {} {} {} ".format(r["url"], r["domain"], r["ip_address"]) if to_return: - return {'results': [{'types': mispattributes['output'], 'values': to_return}]} + return {"results": [{"types": mispattributes["output"], "values": to_return}]} else: - misperrors['error'] = 'Unknown in the EUPI service' + misperrors["error"] = "Unknown in the EUPI service" return misperrors else: - return {'results': [{'types': mispattributes['output'], 'values': ''}]} + return {"results": [{"types": mispattributes["output"], "values": ""}]} def handle_hover(pyeupi, url): try: - result = pyeupi.lookup(url=url)['results'][0] + result = pyeupi.lookup(url=url)["results"][0] except (KeyError, IndexError): - misperrors['error'] = 'Error in EUPI lookup' + misperrors["error"] = "Error in EUPI lookup" return misperrors - return {'results': [{'types': mispattributes['output'], - 'values': result['tag_label'].title()}]} + return {"results": [{"types": mispattributes["output"], "values": result["tag_label"].title()}]} def introspection(): @@ -82,5 +89,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/extract_url_components.py b/misp_modules/modules/expansion/extract_url_components.py index 3f2b09c86..e485e4399 100644 --- a/misp_modules/modules/expansion/extract_url_components.py +++ b/misp_modules/modules/expansion/extract_url_components.py @@ -1,22 +1,24 @@ import json + from pymisp import MISPEvent, MISPObject -from . import check_input_attribute, standard_error_message from pymisp.tools._psl_faup import PSLFaup as Faup -misperrors = {'error': 'Error'} -mispattributes = {'input': ['url'], 'format': 'misp_standard'} +from . import check_input_attribute, standard_error_message + +misperrors = {"error": "Error"} +mispattributes = {"input": ["url"], "format": "misp_standard"} moduleinfo = { - 'version': '1', - 'author': 'MISP Team', - 'description': 'Extract URL components', - 'module-type': ['expansion', 'hover'], - 'name': 'URL Components Extractor', - 'logo': '', - 'requirements': [], - 'features': '', - 'references': [], - 'input': '', - 'output': '', + "version": "1", + "author": "MISP Team", + "description": "Extract URL components", + "module-type": ["expansion", "hover"], + "name": "URL Components Extractor", + "logo": "", + "requirements": [], + "features": "", + "references": [], + "input": "", + "output": "", } moduleconfig = [] @@ -25,52 +27,53 @@ def createObjectFromURL(url): f = Faup() f.decode(url) parsed = f.get() - obj = MISPObject('url') - obj.add_attribute('url', type='url', value=url) - if parsed['tld'] is not None: - obj.add_attribute('tld', type='text', value=parsed['tld']) - if parsed['subdomain'] is not None: - obj.add_attribute('subdomain', type='text', value=parsed['subdomain']) - obj.add_attribute('scheme', type='text', value=parsed['scheme']) - if parsed['resource_path'] is not None: - obj.add_attribute('resource_path', type='text', value=parsed['resource_path']) - if parsed['query_string'] is not None: - obj.add_attribute('query_string', type='text', value=parsed['query_string']) - if parsed['port'] is not None: - obj.add_attribute('port', type='port', value=parsed['port']) - obj.add_attribute('host', type='hostname', value=parsed['host']) - if parsed['fragment'] is not None: - obj.add_attribute('fragment', type='text', value=parsed['fragment']) - obj.add_attribute('domain_without_tld', type='text', value=parsed['domain_without_tld']) - obj.add_attribute('domain', type='domain', value=parsed['domain']) + obj = MISPObject("url") + obj.add_attribute("url", type="url", value=url) + if parsed["tld"] is not None: + obj.add_attribute("tld", type="text", value=parsed["tld"]) + if parsed["subdomain"] is not None: + obj.add_attribute("subdomain", type="text", value=parsed["subdomain"]) + obj.add_attribute("scheme", type="text", value=parsed["scheme"]) + if parsed["resource_path"] is not None: + obj.add_attribute("resource_path", type="text", value=parsed["resource_path"]) + if parsed["query_string"] is not None: + obj.add_attribute("query_string", type="text", value=parsed["query_string"]) + if parsed["port"] is not None: + obj.add_attribute("port", type="port", value=parsed["port"]) + obj.add_attribute("host", type="hostname", value=parsed["host"]) + if parsed["fragment"] is not None: + obj.add_attribute("fragment", type="text", value=parsed["fragment"]) + obj.add_attribute("domain_without_tld", type="text", value=parsed["domain_without_tld"]) + obj.add_attribute("domain", type="domain", value=parsed["domain"]) return obj def createEvent(urlObject, attributeUUID, urlAttribute): mispEvent = MISPEvent() mispEvent.add_attribute(**urlAttribute) - urlObject.add_reference(attributeUUID, 'generated-from') + urlObject.add_reference(attributeUUID, "generated-from") mispEvent.add_object(urlObject) return mispEvent + def handler(q=False): if q is False: return False request = json.loads(q) - if not request.get('attribute') or not check_input_attribute(request['attribute']): - return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} - attribute = request['attribute'] + if not request.get("attribute") or not check_input_attribute(request["attribute"]): + return {"error": f"{standard_error_message}, which should contain at least a type, a value and an uuid."} + attribute = request["attribute"] - if attribute['type'] not in mispattributes['input']: - return {'error': 'Bad attribute type'} + if attribute["type"] not in mispattributes["input"]: + return {"error": "Bad attribute type"} - url = attribute['value'] + url = attribute["value"] urlObject = createObjectFromURL(url) - event = createEvent(urlObject, attribute['uuid'], attribute) + event = createEvent(urlObject, attribute["uuid"], attribute) event = json.loads(event.to_json()) - result = {'results': {'Object': event['Object']}} + result = {"results": {"Object": event["Object"]}} return result @@ -79,5 +82,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/farsight_passivedns.py b/misp_modules/modules/expansion/farsight_passivedns.py index 835a7be9c..cb5235a65 100755 --- a/misp_modules/modules/expansion/farsight_passivedns.py +++ b/misp_modules/modules/expansion/farsight_passivedns.py @@ -1,54 +1,61 @@ -import dnsdb2 import json -from . import check_input_attribute, standard_error_message from datetime import datetime -from pymisp import MISPEvent, MISPObject, Distribution - -misperrors = {'error': 'Error'} -standard_query_input = [ - 'hostname', - 'domain', - 'ip-src', - 'ip-dst' -] + +import dnsdb2 +from pymisp import Distribution, MISPEvent, MISPObject + +from . import check_input_attribute, standard_error_message + +misperrors = {"error": "Error"} +standard_query_input = ["hostname", "domain", "ip-src", "ip-dst"] flex_query_input = [ - 'btc', - 'dkim', - 'email', - 'email-src', - 'email-dst', - 'domain|ip', - 'hex', - 'mac-address', - 'mac-eui-64', - 'other', - 'pattern-filename', - 'target-email', - 'text', - 'uri', - 'url', - 'whois-registrant-email', + "btc", + "dkim", + "email", + "email-src", + "email-dst", + "domain|ip", + "hex", + "mac-address", + "mac-eui-64", + "other", + "pattern-filename", + "target-email", + "text", + "uri", + "url", + "whois-registrant-email", ] mispattributes = { - 'input': standard_query_input + flex_query_input, - 'format': 'misp_standard' + "input": standard_query_input + flex_query_input, + "format": "misp_standard", } moduleinfo = { - 'version': '0.5', - 'author': 'Christophe Vandeplas', - 'description': 'Module to access Farsight DNSDB Passive DNS.', - 'module-type': ['expansion', 'hover'], - 'name': 'Farsight DNSDB Lookup', - 'logo': 'farsight.png', - 'requirements': ['An access to the Farsight Passive DNS API (apikey)'], - 'features': 'This module takes a domain, hostname or IP address MISP attribute as input to query the Farsight Passive DNS API.\n The results of rdata and rrset lookups are then returned and parsed into passive-dns objects.\n\nAn API key is required to submit queries to the API.\n It is also possible to define a custom server URL, and to set a limit of results to get.\n This limit is set for each lookup, which means we can have an up to the limit number of passive-dns objects resulting from an rdata query about an IP address, but an up to the limit number of passive-dns objects for each lookup queries about a domain or a hostname (== twice the limit).', - 'references': ['https://www.farsightsecurity.com/', 'https://docs.dnsdb.info/dnsdb-api/'], - 'input': 'A domain, hostname or IP address MISP attribute.', - 'output': 'Passive-dns objects, resulting from the query on the Farsight Passive DNS API.', + "version": "0.5", + "author": "Christophe Vandeplas", + "description": "Module to access Farsight DNSDB Passive DNS.", + "module-type": ["expansion", "hover"], + "name": "Farsight DNSDB Lookup", + "logo": "farsight.png", + "requirements": ["An access to the Farsight Passive DNS API (apikey)"], + "features": ( + "This module takes a domain, hostname or IP address MISP attribute as input to query the Farsight Passive DNS" + " API.\n The results of rdata and rrset lookups are then returned and parsed into passive-dns objects.\n\nAn" + " API key is required to submit queries to the API.\n It is also possible to define a custom server URL, and" + " to set a limit of results to get.\n This limit is set for each lookup, which means we can have an up to the" + " limit number of passive-dns objects resulting from an rdata query about an IP address, but an up to the limit" + " number of passive-dns objects for each lookup queries about a domain or a hostname (== twice the limit)." + ), + "references": [ + "https://www.farsightsecurity.com/", + "https://docs.dnsdb.info/dnsdb-api/", + ], + "input": "A domain, hostname or IP address MISP attribute.", + "output": "Passive-dns objects, resulting from the query on the Farsight Passive DNS API.", } -moduleconfig = ['apikey', 'server', 'limit', 'flex_queries'] +moduleconfig = ["apikey", "server", "limit", "flex_queries"] -DEFAULT_DNSDB_SERVER = 'https://api.dnsdb.info' +DEFAULT_DNSDB_SERVER = "https://api.dnsdb.info" DEFAULT_LIMIT = 10 DEFAULT_DISTRIBUTION_SETTING = Distribution.your_organisation_only.value TYPE_TO_FEATURE = { @@ -64,72 +71,68 @@ "target-email": "attack target email", "uri": "Uniform Resource Identifier", "url": "Uniform Resource Locator", - "whois-registrant-email": "email of a domain's registrant" + "whois-registrant-email": "email of a domain's registrant", } -TYPE_TO_FEATURE.update( - dict.fromkeys( - ("ip-src", "ip-dst"), - "IP address" - ) -) -TYPE_TO_FEATURE.update( - dict.fromkeys( - ("email", "email-src", "email-dst"), - "email address" - ) -) -TYPE_TO_FEATURE.update( - dict.fromkeys( - ("other", "text"), - "text" - ) -) - - -class FarsightDnsdbParser(): +TYPE_TO_FEATURE.update(dict.fromkeys(("ip-src", "ip-dst"), "IP address")) +TYPE_TO_FEATURE.update(dict.fromkeys(("email", "email-src", "email-dst"), "email address")) +TYPE_TO_FEATURE.update(dict.fromkeys(("other", "text"), "text")) + + +class FarsightDnsdbParser: def __init__(self, attribute): self.attribute = attribute self.misp_event = MISPEvent() self.misp_event.add_attribute(**attribute) self.passivedns_mapping = { - 'bailiwick': {'type': 'domain', 'object_relation': 'bailiwick'}, - 'count': {'type': 'counter', 'object_relation': 'count'}, - 'raw_rdata': {'type': 'text', 'object_relation': 'raw_rdata'}, - 'rdata': {'type': 'text', 'object_relation': 'rdata'}, - 'rrname': {'type': 'text', 'object_relation': 'rrname'}, - 'rrtype': {'type': 'text', 'object_relation': 'rrtype'}, - 'time_first': {'type': 'datetime', 'object_relation': 'time_first'}, - 'time_last': {'type': 'datetime', 'object_relation': 'time_last'}, - 'zone_time_first': {'type': 'datetime', 'object_relation': 'zone_time_first'}, - 'zone_time_last': {'type': 'datetime', 'object_relation': 'zone_time_last'} + "bailiwick": {"type": "domain", "object_relation": "bailiwick"}, + "count": {"type": "counter", "object_relation": "count"}, + "raw_rdata": {"type": "text", "object_relation": "raw_rdata"}, + "rdata": {"type": "text", "object_relation": "rdata"}, + "rrname": {"type": "text", "object_relation": "rrname"}, + "rrtype": {"type": "text", "object_relation": "rrtype"}, + "time_first": {"type": "datetime", "object_relation": "time_first"}, + "time_last": {"type": "datetime", "object_relation": "time_last"}, + "zone_time_first": { + "type": "datetime", + "object_relation": "zone_time_first", + }, + "zone_time_last": {"type": "datetime", "object_relation": "zone_time_last"}, } - self.comment = 'Result from a %s lookup on DNSDB about the %s: %s' + self.comment = "Result from a %s lookup on DNSDB about the %s: %s" def parse_passivedns_results(self, query_response): for query_type, results in query_response.items(): - comment = self.comment % (query_type, TYPE_TO_FEATURE[self.attribute['type']], self.attribute['value']) + comment = self.comment % ( + query_type, + TYPE_TO_FEATURE[self.attribute["type"]], + self.attribute["value"], + ) for result in results: - passivedns_object = MISPObject('passive-dns') + passivedns_object = MISPObject("passive-dns") passivedns_object.distribution = DEFAULT_DISTRIBUTION_SETTING - if result.get('rdata') and isinstance(result['rdata'], list): - for rdata in result.pop('rdata'): - passivedns_object.add_attribute(**self._parse_attribute(comment, 'rdata', rdata)) + if result.get("rdata") and isinstance(result["rdata"], list): + for rdata in result.pop("rdata"): + passivedns_object.add_attribute(**self._parse_attribute(comment, "rdata", rdata)) for feature, value in result.items(): passivedns_object.add_attribute(**self._parse_attribute(comment, feature, value)) - if result.get('time_first'): - passivedns_object.first_seen = result['time_first'] - if result.get('time_last'): - passivedns_object.last_seen = result['time_last'] - passivedns_object.add_reference(self.attribute['uuid'], 'related-to') + if result.get("time_first"): + passivedns_object.first_seen = result["time_first"] + if result.get("time_last"): + passivedns_object.last_seen = result["time_last"] + passivedns_object.add_reference(self.attribute["uuid"], "related-to") self.misp_event.add_object(passivedns_object) def get_results(self): event = json.loads(self.misp_event.to_json()) - results = {key: event[key] for key in ('Attribute', 'Object')} - return {'results': results} + results = {key: event[key] for key in ("Attribute", "Object")} + return {"results": results} def _parse_attribute(self, comment, feature, value): - attribute = {'value': value, 'comment': comment, 'distribution': DEFAULT_DISTRIBUTION_SETTING} + attribute = { + "value": value, + "comment": comment, + "distribution": DEFAULT_DISTRIBUTION_SETTING, + } attribute.update(self.passivedns_mapping[feature]) return attribute @@ -138,28 +141,37 @@ def handler(q=False): if q is False: return False request = json.loads(q) - if not request.get('config') or not request['config'].get('apikey'): - misperrors['error'] = 'Farsight DNSDB apikey is missing' + if not request.get("config") or not request["config"].get("apikey"): + misperrors["error"] = "Farsight DNSDB apikey is missing" return misperrors - if not request.get('attribute') or not check_input_attribute(request['attribute']): - return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} - attribute = request['attribute'] - if attribute['type'] not in mispattributes['input']: - return {'error': 'Unsupported attributes type'} - config = request['config'] - if not config.get('server'): - config['server'] = DEFAULT_DNSDB_SERVER - client_args = {feature: config[feature] for feature in ('apikey', 'server')} + if not request.get("attribute") or not check_input_attribute(request["attribute"]): + return {"error": f"{standard_error_message}, which should contain at least a type, a value and an uuid."} + attribute = request["attribute"] + if attribute["type"] not in mispattributes["input"]: + return {"error": "Unsupported attributes type"} + config = request["config"] + if not config.get("server"): + config["server"] = DEFAULT_DNSDB_SERVER + client_args = {feature: config[feature] for feature in ("apikey", "server")} client = dnsdb2.Client(**client_args) to_query, args = parse_input(attribute, config) try: response = to_query(client, *args) except dnsdb2.DnsdbException as e: - return {'error': e.__str__()} + return {"error": e.__str__()} except dnsdb2.exceptions.QueryError: - return {'error': 'Communication error occurs while executing a query, or the server reports an error due to invalid arguments.'} + return { + "error": ( + "Communication error occurs while executing a query, or the server reports an error due to invalid" + " arguments." + ) + } if not response: - return {'error': f"Empty results on Farsight DNSDB for the {TYPE_TO_FEATURE[attribute['type']]}: {attribute['value']}."} + return { + "error": ( + f"Empty results on Farsight DNSDB for the {TYPE_TO_FEATURE[attribute['type']]}: {attribute['value']}." + ) + } parser = FarsightDnsdbParser(attribute) parser.parse_passivedns_results(response) return parser.get_results() @@ -167,45 +179,45 @@ def handler(q=False): def parse_input(attribute, config): lookup_args = { - 'limit': config['limit'] if config.get('limit') else DEFAULT_LIMIT, - 'offset': 0, - 'ignore_limited': True, - 'humantime': True + "limit": config["limit"] if config.get("limit") else DEFAULT_LIMIT, + "offset": 0, + "ignore_limited": True, + "humantime": True, } - if attribute.get('first_seen'): - lookup_args['time_first_after'] = parse_timestamp(attribute['first_seen']) - attribute_type = attribute['type'] + if attribute.get("first_seen"): + lookup_args["time_first_after"] = parse_timestamp(attribute["first_seen"]) + attribute_type = attribute["type"] if attribute_type in flex_query_input: - return flex_queries, (lookup_args, attribute['value']) - flex = add_flex_queries(config.get('flex_queries')) - to_query = lookup_ip if 'ip-' in attribute_type else lookup_name - return to_query, (lookup_args, attribute['value'], flex) + return flex_queries, (lookup_args, attribute["value"]) + flex = add_flex_queries(config.get("flex_queries")) + to_query = lookup_ip if "ip-" in attribute_type else lookup_name + return to_query, (lookup_args, attribute["value"], flex) def parse_timestamp(str_date): - datetime_date = datetime.strptime(str_date, '%Y-%m-%dT%H:%M:%S.%f%z') + datetime_date = datetime.strptime(str_date, "%Y-%m-%dT%H:%M:%S.%f%z") return str(int(datetime_date.timestamp())) def add_flex_queries(flex): if not flex: return False - if flex in ('True', 'true', True, '1', 1): + if flex in ("True", "true", True, "1", 1): return True return False def flex_queries(client, lookup_args, name): response = {} - name = name.replace('@', '.') - for feature in ('rdata', 'rrnames'): - to_call = getattr(client, f'flex_{feature}_regex') + name = name.replace("@", ".") + for feature in ("rdata", "rrnames"): + to_call = getattr(client, f"flex_{feature}_regex") results = list(to_call(name, **lookup_args)) - for result in list(to_call(name.replace('.', '\\.'), **lookup_args)): + for result in list(to_call(name.replace(".", "\\."), **lookup_args)): if result not in results: results.append(result) if results: - response[f'flex_{feature}'] = results + response[f"flex_{feature}"] = results return response @@ -214,11 +226,11 @@ def lookup_name(client, lookup_args, name, flex): # RRSET = entries in the left-hand side of the domain name related labels rrset_response = list(client.lookup_rrset(name, **lookup_args)) if rrset_response: - response['rrset'] = rrset_response + response["rrset"] = rrset_response # RDATA = entries on the right-hand side of the domain name related labels rdata_response = list(client.lookup_rdata_name(name, **lookup_args)) if rdata_response: - response['rdata'] = rdata_response + response["rdata"] = rdata_response if flex: response.update(flex_queries(client, lookup_args, name)) return response @@ -228,7 +240,7 @@ def lookup_ip(client, lookup_args, ip, flex): response = {} res = list(client.lookup_rdata_ip(ip, **lookup_args)) if res: - response['rdata'] = res + response["rdata"] = res if flex: response.update(flex_queries(client, lookup_args, ip)) return response @@ -239,5 +251,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/geoip_asn.py b/misp_modules/modules/expansion/geoip_asn.py index 16908437e..9469eeed9 100644 --- a/misp_modules/modules/expansion/geoip_asn.py +++ b/misp_modules/modules/expansion/geoip_asn.py @@ -1,33 +1,40 @@ import json -import geoip2.database -import sys import logging +import sys + +import geoip2.database -log = logging.getLogger('geoip_asn') +log = logging.getLogger("geoip_asn") log.setLevel(logging.DEBUG) ch = logging.StreamHandler(sys.stdout) ch.setLevel(logging.DEBUG) -formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") ch.setFormatter(formatter) log.addHandler(ch) -misperrors = {'error': 'Error'} -mispattributes = {'input': ['ip-src', 'ip-dst', 'domain|ip'], 'output': ['freetext']} -moduleconfig = ['local_geolite_db'] +misperrors = {"error": "Error"} +mispattributes = {"input": ["ip-src", "ip-dst", "domain|ip"], "output": ["freetext"]} +moduleconfig = ["local_geolite_db"] # possible module-types: 'expansion', 'hover' or both moduleinfo = { - 'version': '0.1', - 'author': 'GlennHD', - 'description': 'Query a local copy of the Maxmind Geolite ASN database (MMDB format)', - 'module-type': ['expansion', 'hover'], - 'name': 'GeoIP ASN Lookup', - 'logo': 'maxmind.png', - 'requirements': ["A local copy of Maxmind's Geolite database"], - 'features': "The module takes an IP address attribute as input and queries a local copy of the Maxmind's Geolite database to get information about the related AS number.", - 'references': ['https://www.maxmind.com/en/home'], - 'input': 'An IP address MISP attribute.', - 'output': 'Text containing information about the AS number of the IP address.', - 'descrption': "An expansion module to query a local copy of Maxmind's Geolite database with an IP address, in order to get information about its related AS number.", + "version": "0.1", + "author": "GlennHD", + "description": "Query a local copy of the Maxmind Geolite ASN database (MMDB format)", + "module-type": ["expansion", "hover"], + "name": "GeoIP ASN Lookup", + "logo": "maxmind.png", + "requirements": ["A local copy of Maxmind's Geolite database"], + "features": ( + "The module takes an IP address attribute as input and queries a local copy of the Maxmind's Geolite database" + " to get information about the related AS number." + ), + "references": ["https://www.maxmind.com/en/home"], + "input": "An IP address MISP attribute.", + "output": "Text containing information about the AS number of the IP address.", + "descrption": ( + "An expansion module to query a local copy of Maxmind's Geolite database with an IP address, in order to get" + " information about its related AS number." + ), } @@ -36,32 +43,34 @@ def handler(q=False): return False request = json.loads(q) - if not request.get('config') or not request['config'].get('local_geolite_db'): - return {'error': 'Please specify the path of your local copy of the Maxmind Geolite ASN database'} - path_to_geolite = request['config']['local_geolite_db'] + if not request.get("config") or not request["config"].get("local_geolite_db"): + return {"error": "Please specify the path of your local copy of the Maxmind Geolite ASN database"} + path_to_geolite = request["config"]["local_geolite_db"] - if request.get('ip-dst'): - toquery = request['ip-dst'] - elif request.get('ip-src'): - toquery = request['ip-src'] - elif request.get('domain|ip'): - toquery = request['domain|ip'].split('|')[1] + if request.get("ip-dst"): + toquery = request["ip-dst"] + elif request.get("ip-src"): + toquery = request["ip-src"] + elif request.get("domain|ip"): + toquery = request["domain|ip"].split("|")[1] else: return False try: reader = geoip2.database.Reader(path_to_geolite) except FileNotFoundError: - return {'error': f'Unable to locate the GeoLite database you specified ({path_to_geolite}).'} + return {"error": f"Unable to locate the GeoLite database you specified ({path_to_geolite})."} log.debug(toquery) try: answer = reader.asn(toquery) - stringmap = 'ASN=' + str(answer.autonomous_system_number) + ', AS Org=' + str(answer.autonomous_system_organization) + stringmap = ( + "ASN=" + str(answer.autonomous_system_number) + ", AS Org=" + str(answer.autonomous_system_organization) + ) except Exception as e: - misperrors['error'] = f"GeoIP resolving error: {e}" + misperrors["error"] = f"GeoIP resolving error: {e}" return misperrors - r = {'results': [{'types': mispattributes['output'], 'values': stringmap}]} + r = {"results": [{"types": mispattributes["output"], "values": stringmap}]} return r @@ -71,5 +80,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/geoip_city.py b/misp_modules/modules/expansion/geoip_city.py index 643f08584..01a89e2c1 100644 --- a/misp_modules/modules/expansion/geoip_city.py +++ b/misp_modules/modules/expansion/geoip_city.py @@ -1,32 +1,39 @@ import json -import geoip2.database -import sys import logging +import sys + +import geoip2.database -log = logging.getLogger('geoip_city') +log = logging.getLogger("geoip_city") log.setLevel(logging.DEBUG) ch = logging.StreamHandler(sys.stdout) ch.setLevel(logging.DEBUG) -formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") ch.setFormatter(formatter) log.addHandler(ch) -misperrors = {'error': 'Error'} -mispattributes = {'input': ['ip-src', 'ip-dst', 'domain|ip'], 'output': ['freetext']} -moduleconfig = ['local_geolite_db'] +misperrors = {"error": "Error"} +mispattributes = {"input": ["ip-src", "ip-dst", "domain|ip"], "output": ["freetext"]} +moduleconfig = ["local_geolite_db"] # possible module-types: 'expansion', 'hover' or both moduleinfo = { - 'version': '0.1', - 'author': 'GlennHD', - 'description': "An expansion module to query a local copy of Maxmind's Geolite database with an IP address, in order to get information about the city where it is located.", - 'module-type': ['expansion', 'hover'], - 'name': 'GeoIP City Lookup', - 'logo': 'maxmind.png', - 'requirements': ["A local copy of Maxmind's Geolite database"], - 'features': "The module takes an IP address attribute as input and queries a local copy of the Maxmind's Geolite database to get information about the city where this IP address is located.", - 'references': ['https://www.maxmind.com/en/home'], - 'input': 'An IP address MISP attribute.', - 'output': 'Text containing information about the city where the IP address is located.', + "version": "0.1", + "author": "GlennHD", + "description": ( + "An expansion module to query a local copy of Maxmind's Geolite database with an IP address, in order to get" + " information about the city where it is located." + ), + "module-type": ["expansion", "hover"], + "name": "GeoIP City Lookup", + "logo": "maxmind.png", + "requirements": ["A local copy of Maxmind's Geolite database"], + "features": ( + "The module takes an IP address attribute as input and queries a local copy of the Maxmind's Geolite database" + " to get information about the city where this IP address is located." + ), + "references": ["https://www.maxmind.com/en/home"], + "input": "An IP address MISP attribute.", + "output": "Text containing information about the city where the IP address is located.", } @@ -35,33 +42,42 @@ def handler(q=False): return False request = json.loads(q) - if not request.get('config') or not request['config'].get('local_geolite_db'): - return {'error': 'Please specify the path of your local copy of Maxminds Geolite database'} - path_to_geolite = request['config']['local_geolite_db'] + if not request.get("config") or not request["config"].get("local_geolite_db"): + return {"error": "Please specify the path of your local copy of Maxminds Geolite database"} + path_to_geolite = request["config"]["local_geolite_db"] - if request.get('ip-dst'): - toquery = request['ip-dst'] - elif request.get('ip-src'): - toquery = request['ip-src'] - elif request.get('domain|ip'): - toquery = request['domain|ip'].split('|')[1] + if request.get("ip-dst"): + toquery = request["ip-dst"] + elif request.get("ip-src"): + toquery = request["ip-src"] + elif request.get("domain|ip"): + toquery = request["domain|ip"].split("|")[1] else: return False try: reader = geoip2.database.Reader(path_to_geolite) except FileNotFoundError: - return {'error': f'Unable to locate the GeoLite database you specified ({path_to_geolite}).'} + return {"error": f"Unable to locate the GeoLite database you specified ({path_to_geolite})."} log.debug(toquery) try: answer = reader.city(toquery) - stringmap = 'Continent=' + str(answer.continent.name) + ', Country=' + str(answer.country.name) + ', Subdivision=' + str(answer.subdivisions.most_specific.name) + ', City=' + str(answer.city.name) + stringmap = ( + "Continent=" + + str(answer.continent.name) + + ", Country=" + + str(answer.country.name) + + ", Subdivision=" + + str(answer.subdivisions.most_specific.name) + + ", City=" + + str(answer.city.name) + ) except Exception as e: - misperrors['error'] = f"GeoIP resolving error: {e}" + misperrors["error"] = f"GeoIP resolving error: {e}" return misperrors - r = {'results': [{'types': mispattributes['output'], 'values': stringmap}]} + r = {"results": [{"types": mispattributes["output"], "values": stringmap}]} return r @@ -71,5 +87,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/geoip_country.py b/misp_modules/modules/expansion/geoip_country.py index 1b3336dab..85d5654fa 100644 --- a/misp_modules/modules/expansion/geoip_country.py +++ b/misp_modules/modules/expansion/geoip_country.py @@ -1,32 +1,37 @@ import json -import geoip2.database -import sys import logging +import sys + +import geoip2.database -log = logging.getLogger('geoip_country') +log = logging.getLogger("geoip_country") log.setLevel(logging.DEBUG) ch = logging.StreamHandler(sys.stdout) ch.setLevel(logging.DEBUG) -formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") ch.setFormatter(formatter) log.addHandler(ch) -misperrors = {'error': 'Error'} -mispattributes = {'input': ['ip-src', 'ip-dst', 'domain|ip'], 'output': ['freetext']} -moduleconfig = ['local_geolite_db'] +misperrors = {"error": "Error"} +mispattributes = {"input": ["ip-src", "ip-dst", "domain|ip"], "output": ["freetext"]} +moduleconfig = ["local_geolite_db"] # possible module-types: 'expansion', 'hover' or both moduleinfo = { - 'version': '0.2', - 'author': 'Andreas Muehlemann', - 'description': 'Query a local copy of Maxminds Geolite database, updated for MMDB format', - 'module-type': ['expansion', 'hover'], - 'name': 'GeoIP Country Lookup', - 'logo': 'maxmind.png', - 'requirements': ["A local copy of Maxmind's Geolite database"], - 'features': "This module takes an IP address MISP attribute as input and queries a local copy of the Maxmind's Geolite database to get information about the location of this IP address.\n\nPlease note that composite attributes domain|ip are also supported.", - 'references': ['https://www.maxmind.com/en/home'], - 'input': 'An IP address MISP Attribute.', - 'output': 'Text containing information about the location of the IP address.', + "version": "0.2", + "author": "Andreas Muehlemann", + "description": "Query a local copy of Maxminds Geolite database, updated for MMDB format", + "module-type": ["expansion", "hover"], + "name": "GeoIP Country Lookup", + "logo": "maxmind.png", + "requirements": ["A local copy of Maxmind's Geolite database"], + "features": ( + "This module takes an IP address MISP attribute as input and queries a local copy of the Maxmind's Geolite" + " database to get information about the location of this IP address.\n\nPlease note that composite attributes" + " domain|ip are also supported." + ), + "references": ["https://www.maxmind.com/en/home"], + "input": "An IP address MISP Attribute.", + "output": "Text containing information about the location of the IP address.", } @@ -35,31 +40,31 @@ def handler(q=False): return False request = json.loads(q) - if not request.get('config') or not request['config'].get('local_geolite_db'): - return {'error': 'Please specify the path of your local copy of Maxminds Geolite database'} - path_to_geolite = request['config']['local_geolite_db'] + if not request.get("config") or not request["config"].get("local_geolite_db"): + return {"error": "Please specify the path of your local copy of Maxminds Geolite database"} + path_to_geolite = request["config"]["local_geolite_db"] - if request.get('ip-dst'): - toquery = request['ip-dst'] - elif request.get('ip-src'): - toquery = request['ip-src'] - elif request.get('domain|ip'): - toquery = request['domain|ip'].split('|')[1] + if request.get("ip-dst"): + toquery = request["ip-dst"] + elif request.get("ip-src"): + toquery = request["ip-src"] + elif request.get("domain|ip"): + toquery = request["domain|ip"].split("|")[1] else: return False try: reader = geoip2.database.Reader(path_to_geolite) except FileNotFoundError: - return {'error': f'Unable to locate the GeoLite database you specified ({path_to_geolite}).'} + return {"error": f"Unable to locate the GeoLite database you specified ({path_to_geolite})."} log.debug(toquery) try: answer = reader.country(toquery) except Exception as e: - misperrors['error'] = f"GeoIP resolving error: {e}" + misperrors["error"] = f"GeoIP resolving error: {e}" return misperrors - r = {'results': [{'types': mispattributes['output'], 'values': [answer.country.iso_code]}]} + r = {"results": [{"types": mispattributes["output"], "values": [answer.country.iso_code]}]} return r @@ -69,5 +74,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/google_safe_browsing.py b/misp_modules/modules/expansion/google_safe_browsing.py index 84aca8c47..bc9621189 100644 --- a/misp_modules/modules/expansion/google_safe_browsing.py +++ b/misp_modules/modules/expansion/google_safe_browsing.py @@ -1,26 +1,29 @@ # import requests import json -from pymisp import MISPObject, MISPAttribute, MISPEvent -from . import check_input_attribute, checking_error, standard_error_message + +from pymisp import MISPEvent, MISPObject from pysafebrowsing import SafeBrowsing -misperrors = {'error': 'Error'} -mispattributes = {'input': ['url'], 'format': 'misp_standard'} +from . import check_input_attribute, checking_error, standard_error_message + +misperrors = {"error": "Error"} +mispattributes = {"input": ["url"], "format": "misp_standard"} moduleinfo = { - 'version': '0.1', - 'author': 'Stephanie S', - 'description': 'Google safe browsing expansion module', - 'module-type': ['expansion', 'hover'], - 'name': 'Google Safe Browsing Lookup', - 'logo': '', - 'requirements': [], - 'features': '', - 'references': [], - 'input': '', - 'output': '', + "version": "0.1", + "author": "Stephanie S", + "description": "Google safe browsing expansion module", + "module-type": ["expansion", "hover"], + "name": "Google Safe Browsing Lookup", + "logo": "", + "requirements": [], + "features": "", + "references": [], + "input": "", + "output": "", } -moduleconfig = ['api_key'] +moduleconfig = ["api_key"] + def handler(q=False): if q is False: @@ -29,10 +32,10 @@ def handler(q=False): if "config" not in request or "api_key" not in request["config"]: return {"error": "Google Safe Browsing API key is missing"} - if not request.get('attribute') or not check_input_attribute(request['attribute'], requirements=('type', 'value')): - return {'error': f'{standard_error_message}, {checking_error}.'} - if request['attribute']['type'] not in mispattributes['input']: - return {'error': 'Unsupported attribute type.'} + if not request.get("attribute") or not check_input_attribute(request["attribute"], requirements=("type", "value")): + return {"error": f"{standard_error_message}, {checking_error}."} + if request["attribute"]["type"] not in mispattributes["input"]: + return {"error": "Unsupported attribute type."} api_key = request["config"]["api_key"] url = request["attribute"]["value"] @@ -42,45 +45,50 @@ def handler(q=False): response = s.lookup_urls([url]) event = MISPEvent() - obj = MISPObject('google-safe-browsing') - event.add_attribute(**request['attribute']) + obj = MISPObject("google-safe-browsing") + event.add_attribute(**request["attribute"]) - if (response[url]['malicious'] != False): + if response[url]["malicious"] != False: # gsb threat types: THREAT_TYPE_UNSPECIFIED, MALWARE, SOCIAL_ENGINEERING, UNWANTED_SOFTWARE, POTENTIALLY_HARMFUL_APPLICATION - gsb_circl_threat_taxonomy = {"MALWARE": 'malware', "SOCIAL_ENGINEERING": 'social-engineering'} - - threats = response[url]['threats'] - malicious = response[url]['malicious'] - platforms = response[url]['platforms'] - - malicious_attribute = obj.add_attribute('malicious', **{'type': 'boolean', 'value': malicious}) - malicious_attribute.add_tag(f'ioc:artifact-state="malicious"') - threat_attribute = obj.add_attribute('threats', **{'type': 'text', 'value': str(" ".join(threats))}) + gsb_circl_threat_taxonomy = { + "MALWARE": "malware", + "SOCIAL_ENGINEERING": "social-engineering", + } + + threats = response[url]["threats"] + malicious = response[url]["malicious"] + platforms = response[url]["platforms"] + + malicious_attribute = obj.add_attribute("malicious", **{"type": "boolean", "value": malicious}) + malicious_attribute.add_tag('ioc:artifact-state="malicious"') + threat_attribute = obj.add_attribute("threats", **{"type": "text", "value": str(" ".join(threats))}) for threat in threats: # If the threat exists as a key in taxonomy_dict, add that tag - if (gsb_circl_threat_taxonomy.get(threat) is not None): + if gsb_circl_threat_taxonomy.get(threat) is not None: threat_attribute.add_tag(f'circl:incident="{gsb_circl_threat_taxonomy.get(threat)}"') else: - threat_attribute.add_tag(f'threat-type:{str(threat).lower()}') - obj.add_attribute('platforms', **{'type': 'text', 'value': str(" ".join(platforms))}) + threat_attribute.add_tag(f"threat-type:{str(threat).lower()}") + obj.add_attribute("platforms", **{"type": "text", "value": str(" ".join(platforms))}) else: - malicious_attribute = obj.add_attribute('malicious', **{'type': 'boolean', 'value': 0}) # 0 == False - malicious_attribute.add_tag(f'ioc:artifact-state="not-malicious"') + malicious_attribute = obj.add_attribute("malicious", **{"type": "boolean", "value": 0}) # 0 == False + malicious_attribute.add_tag('ioc:artifact-state="not-malicious"') - obj.add_reference(request['attribute']['uuid'], "describes") + obj.add_reference(request["attribute"]["uuid"], "describes") event.add_object(obj) # Avoid serialization issue event = json.loads(event.to_json()) - return {"results": {'Object': event['Object'], 'Attribute': event['Attribute']}} + return {"results": {"Object": event["Object"], "Attribute": event["Attribute"]}} except Exception as error: return {"error": "An error occurred: " + str(error)} + def introspection(): return mispattributes + def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/google_search.py b/misp_modules/modules/expansion/google_search.py deleted file mode 100644 index fd9febe7f..000000000 --- a/misp_modules/modules/expansion/google_search.py +++ /dev/null @@ -1,57 +0,0 @@ -import json -import random -import time -try: - from googleapi import google -except ImportError: - print("GoogleAPI not installed. Command : pip install git+https://github.com/abenassi/Google-Search-API") - -misperrors = {'error': 'Error'} -mispattributes = {'input': ['url'], 'output': ['text']} -moduleinfo = { - 'author': 'Oun & Gindt', - 'module-type': ['hover'], - 'name': 'Google Search', - 'description': 'An expansion hover module to expand google search information about an URL', - 'version': '1.0', - 'logo': 'google.png', - 'requirements': ['The python Google Search API library'], - 'features': 'The module takes an url as input to query the Google search API. The result of the query is then return as raw text.', - 'references': ['https://github.com/abenassi/Google-Search-API'], - 'input': 'An url attribute.', - 'output': 'Text containing the result of a Google search on the input url.', -} - - -def sleep(retry): - time.sleep(random.uniform(0, min(40, 0.01 * 2 ** retry))) - - -def handler(q=False): - if q is False: - return False - request = json.loads(q) - if not request.get('url'): - return {'error': "Unsupported attributes type"} - num_page = 1 - res = "" - # The googleapi module sets a random useragent. The output depends on the useragent. - # It's better to retry 3 times. - for retry in range(3): - search_results = google.search(request['url'], num_page) - if len(search_results) > 0: - break - sleep(retry) - for i, search_result in enumerate(search_results): - res += "("+str(i+1)+")" + '\t' - res += json.dumps(search_result.description, ensure_ascii=False) - res += '\n\n' - return {'results': [{'types': mispattributes['output'], 'values':res}]} - - -def introspection(): - return mispattributes - - -def version(): - return moduleinfo diff --git a/misp_modules/modules/expansion/google_threat_intelligence.py b/misp_modules/modules/expansion/google_threat_intelligence.py index 533618a78..009f402f8 100644 --- a/misp_modules/modules/expansion/google_threat_intelligence.py +++ b/misp_modules/modules/expansion/google_threat_intelligence.py @@ -15,37 +15,55 @@ """Google Threat Intelligence MISP expansion module.""" from urllib import parse -import vt -import pymisp +import pymisp +import vt mispattributes = { - 'input': [ - 'hostname', - 'domain', - 'ip-src', - 'ip-dst', - 'md5', - 'sha1', - 'sha256', - 'url', + "input": [ + "hostname", + "domain", + "ip-src", + "ip-dst", + "md5", + "sha1", + "sha256", + "url", ], - 'format': 'misp_standard', + "format": "misp_standard", } moduleinfo = { - 'version': '2', - 'author': 'Google Threat Intelligence team', - 'description': "An expansion module to have the observable's threat score assessed by Google Threat Intelligence.", - 'module-type': ['expansion'], - 'name': 'Google Threat Intelligence Lookup', - 'config': ['apikey', 'event_limit', 'proxy_host', 'proxy_port', 'proxy_username', 'proxy_password'], - 'logo': 'google_threat_intelligence.png', - 'requirements': ['An access to the Google Threat Intelligence API (apikey), with a high request rate limit.'], - 'features': 'GTI assessment for the given observable, this include information about level of severity, a clear verdict (malicious, suspicious, undetected and benign) and additional information provided by the Mandiant expertise combined with the VirusTotal database.\n\n[Output example screeshot](https://github.com/MISP/MISP/assets/4747608/e275db2f-bb1e-4413-8cc0-ec3cb05e0414)', - 'references': ['https://www.virustotal.com/', 'https://gtidocs.virustotal.com/reference'], - 'input': 'A domain, hash (md5, sha1, sha256 or sha512), hostname or IP address attribute.', - 'output': 'Text fields containing the threat score, the severity, the verdict and the threat label of the observable inspected.', + "version": "2", + "author": "Google Threat Intelligence team", + "description": "An expansion module to have the observable's threat score assessed by Google Threat Intelligence.", + "module-type": ["expansion"], + "name": "Google Threat Intelligence Lookup", + "config": [ + "apikey", + "event_limit", + "proxy_host", + "proxy_port", + "proxy_username", + "proxy_password", + ], + "logo": "google_threat_intelligence.png", + "requirements": ["An access to the Google Threat Intelligence API (apikey), with a high request rate limit."], + "features": ( + "GTI assessment for the given observable, this include information about level of severity, a clear verdict" + " (malicious, suspicious, undetected and benign) and additional information provided by the Mandiant expertise" + " combined with the VirusTotal database.\n\n[Output example" + " screeshot](https://github.com/MISP/MISP/assets/4747608/e275db2f-bb1e-4413-8cc0-ec3cb05e0414)" + ), + "references": [ + "https://www.virustotal.com/", + "https://gtidocs.virustotal.com/reference", + ], + "input": "A domain, hash (md5, sha1, sha256 or sha512), hostname or IP address attribute.", + "output": ( + "Text fields containing the threat score, the severity, the verdict and the threat label of the observable" + " inspected." + ), } DEFAULT_RESULTS_LIMIT = 10 @@ -53,6 +71,7 @@ class GoogleThreatIntelligenceParser: """Main parser class to create the MISP event.""" + def __init__(self, client: vt.Client, limit: int) -> None: self.client = client self.limit = limit or DEFAULT_RESULTS_LIMIT @@ -60,29 +79,26 @@ def __init__(self, client: vt.Client, limit: int) -> None: self.attribute = pymisp.MISPAttribute() self.parsed_objects = {} self.input_types_mapping = { - 'ip-src': self.parse_ip, - 'ip-dst': self.parse_ip, - 'domain': self.parse_domain, - 'hostname': self.parse_domain, - 'md5': self.parse_hash, - 'sha1': self.parse_hash, - 'sha256': self.parse_hash, - 'url': self.parse_url, - 'ip-src|port': self.parse_ip_port, - 'ip-dst|port': self.parse_ip_port, + "ip-src": self.parse_ip, + "ip-dst": self.parse_ip, + "domain": self.parse_domain, + "hostname": self.parse_domain, + "md5": self.parse_hash, + "sha1": self.parse_hash, + "sha256": self.parse_hash, + "url": self.parse_url, + "ip-src|port": self.parse_ip_port, + "ip-dst|port": self.parse_ip_port, } self.proxies = None @staticmethod - def get_total_analysis(analysis: dict, - known_distributors: dict = None) -> int: - """Get total """ + def get_total_analysis(analysis: dict, known_distributors: dict = None) -> int: + """Get total""" if not analysis: return 0 - count = sum([analysis['undetected'], - analysis['suspicious'], - analysis['harmless']]) - return count if known_distributors else count + analysis['malicious'] + count = sum([analysis["undetected"], analysis["suspicious"], analysis["harmless"]]) + return count if known_distributors else count + analysis["malicious"] def query_api(self, attribute: dict) -> None: """Get data from the API and parse it.""" @@ -92,45 +108,38 @@ def query_api(self, attribute: dict) -> None: def get_results(self) -> dict: """Serialize the MISP event.""" event = self.misp_event.to_dict() - results = { - key: event[key] for key in ('Attribute', 'Object') \ - if (key in event and event[key]) - } - return {'results': results} - + results = {key: event[key] for key in ("Attribute", "Object") if (key in event and event[key])} + return {"results": results} def add_gti_report(self, report: vt.Object) -> str: - analysis = report.get('last_analysis_stats') - total = self.get_total_analysis(analysis, - report.get('known_distributors')) - if report.type == 'ip_address': - rtype = 'ip-address' + analysis = report.get("last_analysis_stats") + total = self.get_total_analysis(analysis, report.get("known_distributors")) + if report.type == "ip_address": + rtype = "ip-address" else: rtype = report.type - permalink = f'https://www.virustotal.com/gui/{rtype}/{report.id}' - - gti_object = pymisp.MISPObject('google-threat-intelligence-report') - gti_object.add_attribute('permalink', type='link', value=permalink) - ratio = f"{analysis['malicious']}/{total}" if analysis else '-/-' - gti_object.add_attribute('detection-ratio', - type='text', - value=ratio, - disable_correlation=True) + permalink = f"https://www.virustotal.com/gui/{rtype}/{report.id}" + + gti_object = pymisp.MISPObject("google-threat-intelligence-report") + gti_object.add_attribute("permalink", type="link", value=permalink) + ratio = f"{analysis['malicious']}/{total}" if analysis else "-/-" + gti_object.add_attribute("detection-ratio", type="text", value=ratio, disable_correlation=True) report_dict = report.to_dict() gti_object.add_attribute( - 'threat-score', type='text', - value=get_key(report_dict, - 'attributes.gti_assessment.threat_score.value')) + "threat-score", + type="text", + value=get_key(report_dict, "attributes.gti_assessment.threat_score.value"), + ) gti_object.add_attribute( - 'verdict', type='text', - value=get_key(report_dict, - 'attributes.gti_assessment.verdict.value').replace( - 'VERDICT_', '')) + "verdict", + type="text", + value=get_key(report_dict, "attributes.gti_assessment.verdict.value").replace("VERDICT_", ""), + ) gti_object.add_attribute( - 'severity', type='text', - value=get_key(report_dict, - 'attributes.gti_assessment.severity.value').replace( - 'SEVERITY_', '')) + "severity", + type="text", + value=get_key(report_dict, "attributes.gti_assessment.severity.value").replace("SEVERITY_", ""), + ) self.misp_event.add_object(**gti_object) return gti_object.uuid @@ -138,102 +147,101 @@ def create_misp_object(self, report: vt.Object) -> pymisp.MISPObject: misp_object = None gti_uuid = self.add_gti_report(report) - if report.type == 'file': - misp_object = pymisp.MISPObject('file') - for hash_type in ('md5', 'sha1', 'sha256', 'tlsh', - 'vhash', 'ssdeep', 'imphash'): - misp_object.add_attribute(hash_type, - **{'type': hash_type, - 'value': report.get(hash_type)}) - elif report.type == 'domain': - misp_object = pymisp.MISPObject('domain-ip') - misp_object.add_attribute('domain', type='domain', value=report.id) - elif report.type == 'ip_address': - misp_object = pymisp.MISPObject('domain-ip') - misp_object.add_attribute('ip', type='ip-dst', value=report.id) - elif report.type == 'url': - misp_object = pymisp.MISPObject('url') - misp_object.add_attribute('url', type='url', value=report.id) - misp_object.add_reference(gti_uuid, 'analyzed-with') + if report.type == "file": + misp_object = pymisp.MISPObject("file") + for hash_type in ( + "md5", + "sha1", + "sha256", + "tlsh", + "vhash", + "ssdeep", + "imphash", + ): + misp_object.add_attribute(hash_type, **{"type": hash_type, "value": report.get(hash_type)}) + elif report.type == "domain": + misp_object = pymisp.MISPObject("domain-ip") + misp_object.add_attribute("domain", type="domain", value=report.id) + elif report.type == "ip_address": + misp_object = pymisp.MISPObject("domain-ip") + misp_object.add_attribute("ip", type="ip-dst", value=report.id) + elif report.type == "url": + misp_object = pymisp.MISPObject("url") + misp_object.add_attribute("url", type="url", value=report.id) + misp_object.add_reference(gti_uuid, "analyzed-with") return misp_object def parse_domain(self, domain: str) -> str: - domain_report = self.client.get_object(f'/domains/{domain}') + domain_report = self.client.get_object(f"/domains/{domain}") # DOMAIN domain_object = self.create_misp_object(domain_report) # WHOIS if domain_report.whois: - whois_object = pymisp.MISPObject('whois') - whois_object.add_attribute('text', type='text', - value=domain_report.whois) + whois_object = pymisp.MISPObject("whois") + whois_object.add_attribute("text", type="text", value=domain_report.whois) self.misp_event.add_object(**whois_object) # SIBLINGS AND SUBDOMAINS for relationship_name, misp_name in [ - ('siblings', 'sibling-of'), ('subdomains', 'subdomain')]: - rel_iterator = self.client.iterator( - f'/domains/{domain_report.id}/{relationship_name}', - limit=self.limit) + ("siblings", "sibling-of"), + ("subdomains", "subdomain"), + ]: + rel_iterator = self.client.iterator(f"/domains/{domain_report.id}/{relationship_name}", limit=self.limit) for item in rel_iterator: attr = pymisp.MISPAttribute() - attr.from_dict(**dict(type='domain', value=item.id)) + attr.from_dict(**dict(type="domain", value=item.id)) self.misp_event.add_attribute(**attr) domain_object.add_reference(attr.uuid, misp_name) # RESOLUTIONS - resolutions_iterator = self.client.iterator( - f'/domains/{domain_report.id}/resolutions', limit=self.limit) + resolutions_iterator = self.client.iterator(f"/domains/{domain_report.id}/resolutions", limit=self.limit) for resolution in resolutions_iterator: - domain_object.add_attribute('ip', type='ip-dst', - value=resolution.ip_address) + domain_object.add_attribute("ip", type="ip-dst", value=resolution.ip_address) # COMMUNICATING, DOWNLOADED AND REFERRER FILES for relationship_name, misp_name in [ - ('communicating_files', 'communicates-with'), - ('downloaded_files', 'downloaded-from'), - ('referrer_files', 'referring') + ("communicating_files", "communicates-with"), + ("downloaded_files", "downloaded-from"), + ("referrer_files", "referring"), ]: - files_iterator = self.client.iterator( - f'/domains/{domain_report.id}/{relationship_name}', - limit=self.limit) + files_iterator = self.client.iterator(f"/domains/{domain_report.id}/{relationship_name}", limit=self.limit) for file in files_iterator: file_object = self.create_misp_object(file) file_object.add_reference(domain_object.uuid, misp_name) self.misp_event.add_object(**file_object) # URLS - urls_iterator = self.client.iterator( - f'/domains/{domain_report.id}/urls', limit=self.limit) + urls_iterator = self.client.iterator(f"/domains/{domain_report.id}/urls", limit=self.limit) for url in urls_iterator: url_object = self.create_misp_object(url) - url_object.add_reference(domain_object.uuid, 'hosted-in') + url_object.add_reference(domain_object.uuid, "hosted-in") self.misp_event.add_object(**url_object) self.misp_event.add_object(**domain_object) return domain_object.uuid def parse_hash(self, file_hash: str) -> str: - file_report = self.client.get_object(f'/files/{file_hash}') + file_report = self.client.get_object(f"/files/{file_hash}") file_object = self.create_misp_object(file_report) # ITW URLS - urls_iterator = self.client.iterator( - f'/files/{file_report.id}/itw_urls', limit=self.limit) + urls_iterator = self.client.iterator(f"/files/{file_report.id}/itw_urls", limit=self.limit) for url in urls_iterator: url_object = self.create_misp_object(url) - url_object.add_reference(file_object.uuid, 'downloaded') + url_object.add_reference(file_object.uuid, "downloaded") self.misp_event.add_object(**url_object) # COMMUNICATING, DOWNLOADED AND REFERRER FILES for relationship_name, misp_name in [ - ('contacted_urls', 'communicates-with'), - ('contacted_domains', 'communicates-with'), - ('contacted_ips', 'communicates-with') + ("contacted_urls", "communicates-with"), + ("contacted_domains", "communicates-with"), + ("contacted_ips", "communicates-with"), ]: related_files_iterator = self.client.iterator( - f'/files/{file_report.id}/{relationship_name}', limit=self.limit) + f"/files/{file_report.id}/{relationship_name}", limit=self.limit + ) for related_file in related_files_iterator: related_file_object = self.create_misp_object(related_file) related_file_object.add_reference(file_object.uuid, misp_name) @@ -243,37 +251,32 @@ def parse_hash(self, file_hash: str) -> str: return file_object.uuid def parse_ip_port(self, ipport: str) -> str: - ip = ipport.split('|')[0] + ip = ipport.split("|")[0] self.parse_ip(ip) def parse_ip(self, ip: str) -> str: - ip_report = self.client.get_object(f'/ip_addresses/{ip}') + ip_report = self.client.get_object(f"/ip_addresses/{ip}") # IP ip_object = self.create_misp_object(ip_report) # ASN - asn_object = pymisp.MISPObject('asn') - asn_object.add_attribute('asn', type='AS', value=ip_report.asn) - asn_object.add_attribute('subnet-announced', type='ip-src', - value=ip_report.network) - asn_object.add_attribute('country', type='text', - value=ip_report.country) + asn_object = pymisp.MISPObject("asn") + asn_object.add_attribute("asn", type="AS", value=ip_report.asn) + asn_object.add_attribute("subnet-announced", type="ip-src", value=ip_report.network) + asn_object.add_attribute("country", type="text", value=ip_report.country) self.misp_event.add_object(**asn_object) # RESOLUTIONS - resolutions_iterator = self.client.iterator( - f'/ip_addresses/{ip_report.id}/resolutions', limit=self.limit) + resolutions_iterator = self.client.iterator(f"/ip_addresses/{ip_report.id}/resolutions", limit=self.limit) for resolution in resolutions_iterator: - ip_object.add_attribute('domain', type='domain', - value=resolution.host_name) + ip_object.add_attribute("domain", type="domain", value=resolution.host_name) # URLS - urls_iterator = self.client.iterator( - f'/ip_addresses/{ip_report.id}/urls', limit=self.limit) + urls_iterator = self.client.iterator(f"/ip_addresses/{ip_report.id}/urls", limit=self.limit) for url in urls_iterator: url_object = self.create_misp_object(url) - url_object.add_reference(ip_object.uuid, 'hosted-in') + url_object.add_reference(ip_object.uuid, "hosted-in") self.misp_event.add_object(**url_object) self.misp_event.add_object(**ip_object) @@ -281,17 +284,16 @@ def parse_ip(self, ip: str) -> str: def parse_url(self, url: str) -> str: url_id = vt.url_id(url) - url_report = self.client.get_object(f'/urls/{url_id}') + url_report = self.client.get_object(f"/urls/{url_id}") url_object = self.create_misp_object(url_report) # COMMUNICATING, DOWNLOADED AND REFERRER FILES for relationship_name, misp_name in [ - ('communicating_files', 'communicates-with'), - ('downloaded_files', 'downloaded-from'), - ('referrer_files', 'referring') + ("communicating_files", "communicates-with"), + ("downloaded_files", "downloaded-from"), + ("referrer_files", "referring"), ]: - files_iterator = self.client.iterator( - f'/urls/{url_report.id}/{relationship_name}', limit=self.limit) + files_iterator = self.client.iterator(f"/urls/{url_report.id}/{relationship_name}", limit=self.limit) for file in files_iterator: file_object = self.create_misp_object(file) file_object.add_reference(url_object.uuid, misp_name) @@ -301,10 +303,10 @@ def parse_url(self, url: str) -> str: return url_object.uuid -def get_key(dictionary, key, default_value=''): +def get_key(dictionary, key, default_value=""): """Get value from nested dictionaries.""" dictionary = dictionary or {} - keys = key.split('.') + keys = key.split(".") field_name = keys.pop() for k in keys: if k not in dictionary: @@ -316,75 +318,73 @@ def get_key(dictionary, key, default_value=''): def get_proxy_settings(config: dict) -> dict: """Returns proxy settings in the requests format or None if not set up.""" proxies = None - host = config.get('proxy_host') - port = config.get('proxy_port') - username = config.get('proxy_username') - password = config.get('proxy_password') + host = config.get("proxy_host") + port = config.get("proxy_port") + username = config.get("proxy_username") + password = config.get("proxy_password") if host: if not port: raise KeyError( - ('The google_threat_intelligence_proxy_host config is set, ' - 'please also set the virustotal_proxy_port.')) + "The google_threat_intelligence_proxy_host config is set, please also set the virustotal_proxy_port." + ) parsed = parse.urlparse(host) - if 'http' in parsed.scheme: - scheme = 'http' + if "http" in parsed.scheme: + scheme = "http" else: scheme = parsed.scheme netloc = parsed.netloc - host = f'{netloc}:{port}' + host = f"{netloc}:{port}" if username: if not password: - raise KeyError(('The google_threat_intelligence_' - ' proxy_host config is set, please also' - ' set the virustotal_proxy_password.')) - auth = f'{username}:{password}' - host = auth + '@' + host - - proxies = { - 'http': f'{scheme}://{host}', - 'https': f'{scheme}://{host}' - } + raise KeyError( + "The google_threat_intelligence_" + " proxy_host config is set, please also" + " set the virustotal_proxy_password." + ) + auth = f"{username}:{password}" + host = auth + "@" + host + + proxies = {"http": f"{scheme}://{host}", "https": f"{scheme}://{host}"} return proxies def dict_handler(request: dict): """MISP entry point fo the module.""" - if not request.get('config') or not request['config'].get('apikey'): - return { - 'error': ('A Google Threat Intelligence api ' - 'key is required for this module.') - } + if not request.get("config") or not request["config"].get("apikey"): + return {"error": "A Google Threat Intelligence api key is required for this module."} - if not request.get('attribute'): + if not request.get("attribute"): return { - 'error': ('This module requires an "attribute" field as input,' - ' which should contain at least a type, a value and an' - ' uuid.') + "error": ( + 'This module requires an "attribute" field as input,' + " which should contain at least a type, a value and an" + " uuid." + ) } - if request['attribute']['type'] not in mispattributes['input']: - return {'error': 'Unsupported attribute type.'} + if request["attribute"]["type"] not in mispattributes["input"]: + return {"error": "Unsupported attribute type."} - event_limit = request['config'].get('event_limit') - attribute = request['attribute'] + event_limit = request["config"].get("event_limit") + attribute = request["attribute"] try: - proxy_settings = get_proxy_settings(request.get('config')) + proxy_settings = get_proxy_settings(request.get("config")) client = vt.Client( - request['config']['apikey'], + request["config"]["apikey"], headers={ - 'x-tool': 'MISPModuleGTIExpansion', + "x-tool": "MISPModuleGTIExpansion", }, - proxy=proxy_settings['http'] if proxy_settings else None) - parser = GoogleThreatIntelligenceParser( - client, int(event_limit) if event_limit else None) + proxy=proxy_settings["http"] if proxy_settings else None, + ) + parser = GoogleThreatIntelligenceParser(client, int(event_limit) if event_limit else None) parser.query_api(attribute) except vt.APIError as ex: - return {'error': ex.message} + return {"error": ex.message} except KeyError as ex: - return {'error': str(ex)} + return {"error": str(ex)} return parser.get_results() @@ -399,55 +399,46 @@ def version(): return moduleinfo -if __name__ == '__main__': +if __name__ == "__main__": # Testing/debug calls. import os - api_key = os.getenv('GTI_API_KEY') + + api_key = os.getenv("GTI_API_KEY") # File request_data = { - 'config': {'apikey': api_key}, - 'attribute': { - 'type': 'sha256', - 'value': ('ed01ebfbc9eb5bbea545af4d01bf5f10' - '71661840480439c6e5babe8e080e41aa') - } + "config": {"apikey": api_key}, + "attribute": { + "type": "sha256", + "value": "ed01ebfbc9eb5bbea545af4d01bf5f1071661840480439c6e5babe8e080e41aa", + }, } response = dict_handler(request_data) - report_obj = response['results']['Object'][0] + report_obj = response["results"]["Object"][0] print(report_obj.to_dict()) # URL request_data = { - 'config': {'apikey': api_key}, - 'attribute': { - 'type': 'url', - 'value': 'http://47.21.48.182:60813/Mozi.a' - } + "config": {"apikey": api_key}, + "attribute": {"type": "url", "value": "http://47.21.48.182:60813/Mozi.a"}, } response = dict_handler(request_data) - report_obj = response['results']['Object'][0] + report_obj = response["results"]["Object"][0] print(report_obj.to_dict()) # Ip request_data = { - 'config': {'apikey': api_key}, - 'attribute': { - 'type': 'ip-src', - 'value': '180.72.148.38' - } + "config": {"apikey": api_key}, + "attribute": {"type": "ip-src", "value": "180.72.148.38"}, } response = dict_handler(request_data) - report_obj = response['results']['Object'][0] + report_obj = response["results"]["Object"][0] print(report_obj.to_dict()) # Domain request_data = { - 'config': {'apikey': api_key}, - 'attribute': { - 'type': 'domain', - 'value': 'qexyhuv.com' - } + "config": {"apikey": api_key}, + "attribute": {"type": "domain", "value": "qexyhuv.com"}, } response = dict_handler(request_data) - report_obj = response['results']['Object'][0] + report_obj = response["results"]["Object"][0] print(report_obj.to_dict()) diff --git a/misp_modules/modules/expansion/greynoise.py b/misp_modules/modules/expansion/greynoise.py index 101da0786..7efe61fc0 100644 --- a/misp_modules/modules/expansion/greynoise.py +++ b/misp_modules/modules/expansion/greynoise.py @@ -2,11 +2,7 @@ import json import logging - -try: - from greynoise import GreyNoise -except ImportError: - print("greynoise module not installed.") +from greynoise import GreyNoise from pymisp import MISPAttribute, MISPEvent, MISPObject from . import check_input_attribute, standard_error_message @@ -15,19 +11,33 @@ logger.setLevel(logging.INFO) misperrors = {"error": "Error"} -mispattributes = {"input": ["ip-src", "ip-dst", "vulnerability"], "format": "misp_standard"} +mispattributes = { + "input": ["ip-src", "ip-dst", "vulnerability"], + "format": "misp_standard", +} moduleinfo = { - 'version': '1.2', - 'author': 'Brad Chiappetta ', - 'description': 'Module to query IP and CVE information from GreyNoise', - 'module-type': ['expansion', 'hover'], - 'name': 'GreyNoise Lookup', - 'logo': 'greynoise.png', - 'requirements': ['A Greynoise API key. Both Enterprise (Paid) and Community (Free) API keys are supported, however Community API users will only be able to perform IP lookups.'], - 'features': 'This module supports: 1) Query an IP from GreyNoise to see if it is internet background noise or a common business service 2) Query a CVE from GreyNoise to see the total number of internet scanners looking for the CVE in the last 7 days.', - 'references': ['https://greynoise.io/', 'https://docs.greyniose.io/', 'https://www.greynoise.io/viz/account/'], - 'input': 'An IP address or CVE ID', - 'output': 'IP Lookup information or CVE scanning profile for past 7 days', + "version": "1.2", + "author": "Brad Chiappetta ", + "description": "Module to query IP and CVE information from GreyNoise", + "module-type": ["expansion", "hover"], + "name": "GreyNoise Lookup", + "logo": "greynoise.png", + "requirements": [ + "A Greynoise API key. Both Enterprise (Paid) and Community (Free) API keys are supported, however Community API" + " users will only be able to perform IP lookups." + ], + "features": ( + "This module supports: 1) Query an IP from GreyNoise to see if it is internet background noise or a common" + " business service 2) Query a CVE from GreyNoise to see the total number of internet scanners looking for the" + " CVE in the last 7 days." + ), + "references": [ + "https://greynoise.io/", + "https://docs.greyniose.io/", + "https://www.greynoise.io/viz/account/", + ], + "input": "An IP address or CVE ID", + "output": "IP Lookup information or CVE scanning profile for past 7 days", } moduleconfig = ["api_key", "api_type"] @@ -76,8 +86,14 @@ def __init__(self, attribute): "country_code": {"type": "text", "object_relation": "country-code"}, "country": {"type": "text", "object_relation": "country"}, "organization": {"type": "text", "object_relation": "organization"}, - "destination_country_codes": {"type": "text", "object_relation": "destination-country-codes"}, - "destination_countries": {"type": "text", "object_relation": "destination-countries"}, + "destination_country_codes": { + "type": "text", + "object_relation": "destination-country-codes", + }, + "destination_countries": { + "type": "text", + "object_relation": "destination-countries", + }, "category": {"type": "text", "object_relation": "category"}, "rdns": {"type": "text", "object_relation": "rdns"}, } @@ -141,7 +157,10 @@ def query_greynoise_ip_hover(self, api_key, api_type): for feature, mapping in self.ip_address_metadata_mapping.items(): logger.debug(f"Checking metadata feature {feature}") if response["metadata"].get(feature): - if feature in ["destination_countries", "destination_country_codes"]: + if feature in [ + "destination_countries", + "destination_country_codes", + ]: response["metadata"][feature] = ", ".join(response["metadata"][feature]) attribute = {"value": response["metadata"][feature]} logger.debug(f"Adding Feature: {feature}, Attribute: {attribute}") @@ -327,7 +346,9 @@ def handler(q=False): if "persistent" in request: greynoise_parser.query_greynoise_ip_hover(request["config"]["api_key"], request["config"]["api_type"]) else: - greynoise_parser.query_greynoise_ip_expansion(request["config"]["api_key"], request["config"]["api_type"]) + greynoise_parser.query_greynoise_ip_expansion( + request["config"]["api_key"], request["config"]["api_type"] + ) except ValueError: return {"error": "Not a valid IPv4 address"} diff --git a/misp_modules/modules/expansion/hashdd.py b/misp_modules/modules/expansion/hashdd.py index 310cfb70e..c08cb8294 100755 --- a/misp_modules/modules/expansion/hashdd.py +++ b/misp_modules/modules/expansion/hashdd.py @@ -1,23 +1,27 @@ import json + import requests -misperrors = {'error': 'Error'} -mispattributes = {'input': ['md5'], 'output': ['text']} +misperrors = {"error": "Error"} +mispattributes = {"input": ["md5"], "output": ["text"]} moduleinfo = { - 'version': '0.2', - 'author': 'Alexandre Dulaunoy', - 'description': 'A hover module to check hashes against hashdd.com including NSLR dataset.', - 'module-type': ['hover'], - 'name': 'Hashdd Lookup', - 'logo': '', - 'requirements': [], - 'features': 'This module takes a hash attribute as input to check its known level, using the hashdd API. This information is then displayed.', - 'references': ['https://hashdd.com/'], - 'input': 'A hash MISP attribute (md5).', - 'output': 'Text describing the known level of the hash in the hashdd databases.', + "version": "0.2", + "author": "Alexandre Dulaunoy", + "description": "A hover module to check hashes against hashdd.com including NSLR dataset.", + "module-type": ["hover"], + "name": "Hashdd Lookup", + "logo": "", + "requirements": [], + "features": ( + "This module takes a hash attribute as input to check its known level, using the hashdd API. This information" + " is then displayed." + ), + "references": ["https://hashdd.com/"], + "input": "A hash MISP attribute (md5).", + "output": "Text describing the known level of the hash in the hashdd databases.", } moduleconfig = [] -hashddapi_url = 'https://api.hashdd.com/v1/knownlevel/nsrl/' +hashddapi_url = "https://api.hashdd.com/v1/knownlevel/nsrl/" def handler(q=False): @@ -25,22 +29,22 @@ def handler(q=False): return False v = None request = json.loads(q) - for input_type in mispattributes['input']: + for input_type in mispattributes["input"]: if request.get(input_type): v = request[input_type].upper() break if v is None: - misperrors['error'] = 'Hash value is missing.' + misperrors["error"] = "Hash value is missing." return misperrors r = requests.get(hashddapi_url + v) if r.status_code == 200: state = json.loads(r.text) - summary = state['knownlevel'] if state and state['result'] == "SUCCESS" else state['message'] + summary = state["knownlevel"] if state and state["result"] == "SUCCESS" else state["message"] else: - misperrors['error'] = '{} API not accessible'.format(hashddapi_url) - return misperrors['error'] + misperrors["error"] = "{} API not accessible".format(hashddapi_url) + return misperrors["error"] - r = {'results': [{'types': mispattributes['output'], 'values': summary}]} + r = {"results": [{"types": mispattributes["output"], "values": summary}]} return r @@ -49,5 +53,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/hashlookup.py b/misp_modules/modules/expansion/hashlookup.py index 1752ced94..ed8951fce 100644 --- a/misp_modules/modules/expansion/hashlookup.py +++ b/misp_modules/modules/expansion/hashlookup.py @@ -1,29 +1,39 @@ import json -import requests -from . import check_input_attribute, standard_error_message from collections import defaultdict + +import requests from pymisp import MISPEvent, MISPObject -misperrors = {'error': 'Error'} -mispattributes = {'input': ['md5', 'sha1', 'sha256'], 'format': 'misp_standard'} +from . import check_input_attribute, standard_error_message + +misperrors = {"error": "Error"} +mispattributes = {"input": ["md5", "sha1", "sha256"], "format": "misp_standard"} moduleinfo = { - 'version': '2', - 'author': 'Alexandre Dulaunoy', - 'description': 'An expansion module to query the CIRCL hashlookup services to find it if a hash is part of a known set such as NSRL.', - 'module-type': ['expansion', 'hover'], - 'name': 'CIRCL Hashlookup Lookup', - 'logo': 'circl.png', - 'requirements': [], - 'features': 'The module takes file hashes as input such as a MD5 or SHA1.\n It queries the public CIRCL.lu hashlookup service and return all the hits if the hashes are known in an existing dataset. The module can be configured with a custom hashlookup url if required.\n The module can be used an hover module but also an expansion model to add related MISP objects.\n', - 'references': ['https://www.circl.lu/services/hashlookup/'], - 'input': 'File hashes (MD5, SHA1)', - 'output': 'Object with the filename associated hashes if the hash is part of a known set.', + "version": "2", + "author": "Alexandre Dulaunoy", + "description": ( + "An expansion module to query the CIRCL hashlookup services to find it if a hash is part of a known set such as" + " NSRL." + ), + "module-type": ["expansion", "hover"], + "name": "CIRCL Hashlookup Lookup", + "logo": "circl.png", + "requirements": [], + "features": ( + "The module takes file hashes as input such as a MD5 or SHA1.\n It queries the public CIRCL.lu hashlookup" + " service and return all the hits if the hashes are known in an existing dataset. The module can be configured" + " with a custom hashlookup url if required.\n The module can be used an hover module but also an expansion" + " model to add related MISP objects.\n" + ), + "references": ["https://www.circl.lu/services/hashlookup/"], + "input": "File hashes (MD5, SHA1)", + "output": "Object with the filename associated hashes if the hash is part of a known set.", } moduleconfig = ["custom_API"] -hashlookup_url = 'https://hashlookup.circl.lu/' +hashlookup_url = "https://hashlookup.circl.lu/" -class HashlookupParser(): +class HashlookupParser: def __init__(self, attribute, hashlookupresult, api_url): self.attribute = attribute self.hashlookupresult = hashlookupresult @@ -36,30 +46,42 @@ def get_result(self): if self.references: self.__build_references() event = json.loads(self.misp_event.to_json()) - results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} - return {'results': results} + results = {key: event[key] for key in ("Attribute", "Object") if (key in event and event[key])} + return {"results": results} def parse_hashlookup_information(self): - hashlookup_object = MISPObject('hashlookup') - if 'source' in self.hashlookupresult: - hashlookup_object.add_attribute('source', **{'type': 'text', 'value': self.hashlookupresult['source']}) - if 'KnownMalicious' in self.hashlookupresult: - hashlookup_object.add_attribute('KnownMalicious', **{'type': 'text', 'value': self.hashlookupresult['KnownMalicious']}) - if 'MD5' in self.hashlookupresult: - hashlookup_object.add_attribute('MD5', **{'type': 'md5', 'value': self.hashlookupresult['MD5']}) + hashlookup_object = MISPObject("hashlookup") + if "source" in self.hashlookupresult: + hashlookup_object.add_attribute("source", **{"type": "text", "value": self.hashlookupresult["source"]}) + if "KnownMalicious" in self.hashlookupresult: + hashlookup_object.add_attribute( + "KnownMalicious", + **{"type": "text", "value": self.hashlookupresult["KnownMalicious"]}, + ) + if "MD5" in self.hashlookupresult: + hashlookup_object.add_attribute("MD5", **{"type": "md5", "value": self.hashlookupresult["MD5"]}) # SHA-1 is the default value in hashlookup it must always be present - hashlookup_object.add_attribute('SHA-1', **{'type': 'sha1', 'value': self.hashlookupresult['SHA-1']}) - if 'SHA-256' in self.hashlookupresult: - hashlookup_object.add_attribute('SHA-256', **{'type': 'sha256', 'value': self.hashlookupresult['SHA-256']}) - if 'SSDEEP' in self.hashlookupresult: - hashlookup_object.add_attribute('SSDEEP', **{'type': 'ssdeep', 'value': self.hashlookupresult['SSDEEP']}) - if 'TLSH' in self.hashlookupresult: - hashlookup_object.add_attribute('TLSH', **{'type': 'tlsh', 'value': self.hashlookupresult['TLSH']}) - if 'FileName' in self.hashlookupresult: - hashlookup_object.add_attribute('FileName', **{'type': 'filename', 'value': self.hashlookupresult['FileName']}) - if 'FileSize' in self.hashlookupresult: - hashlookup_object.add_attribute('FileSize', **{'type': 'size-in-bytes', 'value': self.hashlookupresult['FileSize']}) - hashlookup_object.add_reference(self.attribute['uuid'], 'related-to') + hashlookup_object.add_attribute("SHA-1", **{"type": "sha1", "value": self.hashlookupresult["SHA-1"]}) + if "SHA-256" in self.hashlookupresult: + hashlookup_object.add_attribute( + "SHA-256", + **{"type": "sha256", "value": self.hashlookupresult["SHA-256"]}, + ) + if "SSDEEP" in self.hashlookupresult: + hashlookup_object.add_attribute("SSDEEP", **{"type": "ssdeep", "value": self.hashlookupresult["SSDEEP"]}) + if "TLSH" in self.hashlookupresult: + hashlookup_object.add_attribute("TLSH", **{"type": "tlsh", "value": self.hashlookupresult["TLSH"]}) + if "FileName" in self.hashlookupresult: + hashlookup_object.add_attribute( + "FileName", + **{"type": "filename", "value": self.hashlookupresult["FileName"]}, + ) + if "FileSize" in self.hashlookupresult: + hashlookup_object.add_attribute( + "FileSize", + **{"type": "size-in-bytes", "value": self.hashlookupresult["FileSize"]}, + ) + hashlookup_object.add_reference(self.attribute["uuid"], "related-to") self.misp_event.add_object(hashlookup_object) def __build_references(self): @@ -70,38 +92,39 @@ def __build_references(self): misp_object.add_reference(**reference) break + def check_url(url): - return "{}/".format(url) if not url.endswith('/') else url + return "{}/".format(url) if not url.endswith("/") else url def handler(q=False): if q is False: return False request = json.loads(q) - if not request.get('attribute') or not check_input_attribute(request['attribute']): - return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} - attribute = request['attribute'] - if attribute.get('type') == 'md5': + if not request.get("attribute") or not check_input_attribute(request["attribute"]): + return {"error": f"{standard_error_message}, which should contain at least a type, a value and an uuid."} + attribute = request["attribute"] + if attribute.get("type") == "md5": pass - elif attribute.get('type') == 'sha1': + elif attribute.get("type") == "sha1": pass - elif attribute.get('type') == 'sha256': + elif attribute.get("type") == "sha256": pass else: - misperrors['error'] = 'md5 or sha1 or sha256 is missing.' + misperrors["error"] = "md5 or sha1 or sha256 is missing." return misperrors - api_url = check_url(request['config']['custom_API']) if request['config'].get('custom_API') else hashlookup_url - r = requests.get("{}/lookup/{}/{}".format(api_url, attribute.get('type'), attribute['value'])) + api_url = check_url(request["config"]["custom_API"]) if request["config"].get("custom_API") else hashlookup_url + r = requests.get("{}/lookup/{}/{}".format(api_url, attribute.get("type"), attribute["value"])) if r.status_code == 200: hashlookupresult = r.json() if not hashlookupresult: - misperrors['error'] = 'Empty result' + misperrors["error"] = "Empty result" return misperrors elif r.status_code == 404: - misperrors['error'] = 'Non existing hash' + misperrors["error"] = "Non existing hash" return misperrors else: - misperrors['error'] = 'API not accessible' + misperrors["error"] = "API not accessible" return misperrors parser = HashlookupParser(attribute, hashlookupresult, api_url) parser.parse_hashlookup_information() @@ -114,5 +137,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/hibp.py b/misp_modules/modules/expansion/hibp.py index 60a12347e..38111eb76 100644 --- a/misp_modules/modules/expansion/hibp.py +++ b/misp_modules/modules/expansion/hibp.py @@ -1,25 +1,30 @@ # -*- coding: utf-8 -*- -import requests import json -misperrors = {'error': 'Error'} -mispattributes = {'input': ['email-dst', 'email-src'], 'output': ['text']} +import requests + +misperrors = {"error": "Error"} +mispattributes = {"input": ["email-dst", "email-src"], "output": ["text"]} moduleinfo = { - 'version': '0.2', - 'author': 'Corsin Camichel, Aurélien Schwab', - 'description': 'Module to access haveibeenpwned.com API.', - 'module-type': ['hover'], - 'name': 'Have I Been Pwned Lookup', - 'logo': 'hibp.png', - 'requirements': [], - 'features': 'The module takes an email address as input and queries haveibeenpwned.com API to find additional information about it. This additional information actually tells if any account using the email address has already been compromised in a data breach.', - 'references': ['https://haveibeenpwned.com/'], - 'input': 'An email address', - 'output': 'Additional information about the email address.', + "version": "0.2", + "author": "Corsin Camichel, Aurélien Schwab", + "description": "Module to access haveibeenpwned.com API.", + "module-type": ["hover"], + "name": "Have I Been Pwned Lookup", + "logo": "hibp.png", + "requirements": [], + "features": ( + "The module takes an email address as input and queries haveibeenpwned.com API to find additional information" + " about it. This additional information actually tells if any account using the email address has already been" + " compromised in a data breach." + ), + "references": ["https://haveibeenpwned.com/"], + "input": "An email address", + "output": "Additional information about the email address.", } -moduleconfig = ['api_key'] +moduleconfig = ["api_key"] -haveibeenpwned_api_url = 'https://haveibeenpwned.com/api/v3/breachedaccount/' +haveibeenpwned_api_url = "https://haveibeenpwned.com/api/v3/breachedaccount/" API_KEY = "" # details at https://www.troyhunt.com/authentication-and-the-have-i-been-pwned-api/ @@ -27,30 +32,30 @@ def handler(q=False): if q is False: return False request = json.loads(q) - for input_type in mispattributes['input']: + for input_type in mispattributes["input"]: if input_type in request: email = request[input_type] break else: - misperrors['error'] = "Unsupported attributes type" + misperrors["error"] = "Unsupported attributes type" return misperrors - if request.get('config') is None or request['config'].get('api_key') is None: - misperrors['error'] = 'Have I Been Pwned authentication is incomplete (no API key)' + if request.get("config") is None or request["config"].get("api_key") is None: + misperrors["error"] = "Have I Been Pwned authentication is incomplete (no API key)" return misperrors else: - API_KEY = request['config'].get('api_key') + API_KEY = request["config"].get("api_key") - r = requests.get(haveibeenpwned_api_url + email, headers={'hibp-api-key': API_KEY}) + r = requests.get(haveibeenpwned_api_url + email, headers={"hibp-api-key": API_KEY}) if r.status_code == 200: breaches = json.loads(r.text) if breaches: - return {'results': [{'types': mispattributes['output'], 'values': breaches}]} + return {"results": [{"types": mispattributes["output"], "values": breaches}]} elif r.status_code == 404: - return {'results': [{'types': mispattributes['output'], 'values': 'OK (Not Found)'}]} + return {"results": [{"types": mispattributes["output"], "values": "OK (Not Found)"}]} else: - misperrors['error'] = f'haveibeenpwned.com API not accessible (HTTP {str(r.status_code)})' - return misperrors['error'] + misperrors["error"] = f"haveibeenpwned.com API not accessible (HTTP {str(r.status_code)})" + return misperrors["error"] def introspection(): @@ -58,5 +63,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/html_to_markdown.py b/misp_modules/modules/expansion/html_to_markdown.py index 629f59356..8e2ab68d2 100755 --- a/misp_modules/modules/expansion/html_to_markdown.py +++ b/misp_modules/modules/expansion/html_to_markdown.py @@ -1,22 +1,26 @@ import json + import requests -from markdownify import markdownify from bs4 import BeautifulSoup +from markdownify import markdownify -misperrors = {'error': 'Error'} -mispattributes = {'input': ['url'], 'output': ['text']} +misperrors = {"error": "Error"} +mispattributes = {"input": ["url"], "output": ["text"]} moduleinfo = { - 'version': '0.1', - 'author': 'Sami Mokaddem', - 'description': 'Expansion module to fetch the html content from an url and convert it into markdown.', - 'module-type': ['expansion'], - 'name': 'HTML to Markdown', - 'logo': '', - 'requirements': ['The markdownify python library'], - 'features': 'The module take an URL as input and the HTML content is fetched from it. This content is then converted into markdown that is returned as text.', - 'references': [], - 'input': 'URL attribute.', - 'output': 'Markdown content converted from the HTML fetched from the url.', + "version": "0.1", + "author": "Sami Mokaddem", + "description": "Expansion module to fetch the html content from an url and convert it into markdown.", + "module-type": ["expansion"], + "name": "HTML to Markdown", + "logo": "", + "requirements": ["The markdownify python library"], + "features": ( + "The module take an URL as input and the HTML content is fetched from it. This content is then converted into" + " markdown that is returned as text." + ), + "references": [], + "input": "URL attribute.", + "output": "Markdown content converted from the HTML fetched from the url.", } @@ -26,32 +30,31 @@ def fetchHTML(url): def stripUselessTags(html): - soup = BeautifulSoup(html, 'html.parser') - toRemove = ['script', 'head', 'header', 'footer', 'meta', 'link'] + soup = BeautifulSoup(html, "html.parser") + toRemove = ["script", "head", "header", "footer", "meta", "link"] for tag in soup.find_all(toRemove): tag.decompose() return str(soup) def convertHTML(html): - toStrip = ['a', 'img'] - return markdownify(html, heading_style='ATX', strip=toStrip) + toStrip = ["a", "img"] + return markdownify(html, heading_style="ATX", strip=toStrip) def handler(q=False): if q is False: return False request = json.loads(q) - if request.get('url'): - url = request['url'] + if request.get("url"): + url = request["url"] else: return False html = fetchHTML(url) html = stripUselessTags(html) markdown = convertHTML(html) - r = {'results': [{'types': mispattributes['output'], - 'values':[str(markdown)]}]} + r = {"results": [{"types": mispattributes["output"], "values": [str(markdown)]}]} return r diff --git a/misp_modules/modules/expansion/hyasinsight.py b/misp_modules/modules/expansion/hyasinsight.py index 9d1a69702..3e01fce3e 100644 --- a/misp_modules/modules/expansion/hyasinsight.py +++ b/misp_modules/modules/expansion/hyasinsight.py @@ -1,136 +1,160 @@ import json import logging -from typing import Dict, List, Any +import re +from typing import Any, Dict, List import requests -import re -from requests.exceptions import ( - HTTPError, - ProxyError, - InvalidURL, - ConnectTimeout -) +from pymisp import Distribution, MISPEvent, MISPObject +from requests.exceptions import ConnectTimeout, HTTPError, InvalidURL, ProxyError + from . import check_input_attribute, standard_error_message -from pymisp import MISPEvent, MISPObject, Distribution -ip_query_input_type = [ - 'ip-src', - 'ip-dst' -] -domain_query_input_type = [ - 'hostname', - 'domain' -] +ip_query_input_type = ["ip-src", "ip-dst"] +domain_query_input_type = ["hostname", "domain"] email_query_input_type = [ - 'email', - 'email-src', - 'email-dst', - 'target-email', - 'whois-registrant-email' -] -phone_query_input_type = [ - 'phone-number', - 'whois-registrant-phone' + "email", + "email-src", + "email-dst", + "target-email", + "whois-registrant-email", ] +phone_query_input_type = ["phone-number", "whois-registrant-phone"] md5_query_input_type = [ - 'md5', - 'x509-fingerprint-md5', - 'ja3-fingerprint-md5', - 'hassh-md5', - 'hasshserver-md5' + "md5", + "x509-fingerprint-md5", + "ja3-fingerprint-md5", + "hassh-md5", + "hasshserver-md5", ] -sha1_query_input_type = [ - 'sha1', - 'x509-fingerprint-sha1' -] +sha1_query_input_type = ["sha1", "x509-fingerprint-sha1"] -sha256_query_input_type = [ - 'sha256', - 'x509-fingerprint-sha256' -] +sha256_query_input_type = ["sha256", "x509-fingerprint-sha256"] -sha512_query_input_type = [ - 'sha512' -] +sha512_query_input_type = ["sha512"] -misperrors = { - 'error': 'Error' -} +misperrors = {"error": "Error"} mispattributes = { - 'input': ip_query_input_type + domain_query_input_type + email_query_input_type + phone_query_input_type - + md5_query_input_type + sha1_query_input_type + sha256_query_input_type + sha512_query_input_type, - 'format': 'misp_standard' + "input": ( + ip_query_input_type + + domain_query_input_type + + email_query_input_type + + phone_query_input_type + + md5_query_input_type + + sha1_query_input_type + + sha256_query_input_type + + sha512_query_input_type + ), + "format": "misp_standard", } moduleinfo = { - 'version': '0.1', - 'author': 'Mike Champ', - 'description': 'HYAS Insight integration to MISP provides direct, high volume access to HYAS Insight data. It enables investigators and analysts to understand and defend against cyber adversaries and their infrastructure.', - 'module-type': ['expansion', 'hover'], - 'name': 'HYAS Insight Lookup', - 'logo': 'hyas.png', - 'requirements': ['A HYAS Insight API Key.'], - 'features': 'This Module takes the IP Address, Domain, URL, Email, Phone Number, MD5, SHA1, Sha256, SHA512 MISP Attributes as input to query the HYAS Insight API.\n The results of the HYAS Insight API are than are then returned and parsed into Hyas Insight Objects. \n\nAn API key is required to submit queries to the HYAS Insight API.\n', - 'references': ['https://www.hyas.com/hyas-insight/'], - 'input': 'A MISP attribute of type IP Address(ip-src, ip-dst), Domain(hostname, domain), Email Address(email, email-src, email-dst, target-email, whois-registrant-email), Phone Number(phone-number, whois-registrant-phone), MDS(md5, x509-fingerprint-md5, ja3-fingerprint-md5, hassh-md5, hasshserver-md5), SHA1(sha1, x509-fingerprint-sha1), SHA256(sha256, x509-fingerprint-sha256), SHA512(sha512)', - 'output': 'Hyas Insight objects, resulting from the query on the HYAS Insight API.', + "version": "0.1", + "author": "Mike Champ", + "description": ( + "HYAS Insight integration to MISP provides direct, high volume access to HYAS Insight data. It enables" + " investigators and analysts to understand and defend against cyber adversaries and their infrastructure." + ), + "module-type": ["expansion", "hover"], + "name": "HYAS Insight Lookup", + "logo": "hyas.png", + "requirements": ["A HYAS Insight API Key."], + "features": ( + "This Module takes the IP Address, Domain, URL, Email, Phone Number, MD5, SHA1, Sha256, SHA512 MISP Attributes" + " as input to query the HYAS Insight API.\n The results of the HYAS Insight API are than are then returned and" + " parsed into Hyas Insight Objects. \n\nAn API key is required to submit queries to the HYAS Insight API.\n" + ), + "references": ["https://www.hyas.com/hyas-insight/"], + "input": ( + "A MISP attribute of type IP Address(ip-src, ip-dst), Domain(hostname, domain), Email Address(email, email-src," + " email-dst, target-email, whois-registrant-email), Phone Number(phone-number, whois-registrant-phone)," + " MDS(md5, x509-fingerprint-md5, ja3-fingerprint-md5, hassh-md5, hasshserver-md5), SHA1(sha1," + " x509-fingerprint-sha1), SHA256(sha256, x509-fingerprint-sha256), SHA512(sha512)" + ), + "output": "Hyas Insight objects, resulting from the query on the HYAS Insight API.", } -moduleconfig = ['apikey'] +moduleconfig = ["apikey"] TIMEOUT = 60 -logger = logging.getLogger('hyasinsight') +logger = logging.getLogger("hyasinsight") logger.setLevel(logging.DEBUG) -HYAS_API_BASE_URL = 'https://insight.hyas.com/api/ext/' -WHOIS_CURRENT_BASE_URL = 'https://api.hyas.com/' +HYAS_API_BASE_URL = "https://insight.hyas.com/api/ext/" +WHOIS_CURRENT_BASE_URL = "https://api.hyas.com/" DEFAULT_DISTRIBUTION_SETTING = Distribution.your_organisation_only.value -IPV4_REGEX = r'\b((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b([^\/]|$)' -IPV6_REGEX = r'\b(?:(?:[0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,7}:|(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|(?:[0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|(?:[0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|(?:[0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:(?:(:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))\b' # noqa: E501 +IPV4_REGEX = r"\b((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b([^\/]|$)" +IPV6_REGEX = r"\b(?:(?:[0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,7}:|(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|(?:[0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|(?:[0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|(?:[0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:(?:(:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))\b" # noqa: E501 # Enrichment Types # HYAS API endpoints -PASSIVE_DNS_ENDPOINT = 'passivedns' -DYNAMIC_DNS_ENDPOINT = 'dynamicdns' -PASSIVE_HASH_ENDPOINT = 'passivehash' -SINKHOLE_ENDPOINT = 'sinkhole' -SSL_CERTIFICATE_ENDPOINT = 'ssl_certificate' -DEVICE_GEO_ENDPOINT = 'device_geo' -WHOIS_HISTORIC_ENDPOINT = 'whois' -WHOIS_CURRENT_ENDPOINT = 'whois/v1' -MALWARE_RECORDS_ENDPOINT = 'sample' -MALWARE_INFORMATION_ENDPOINT = 'sample/information' -C2ATTRIBUTION_ENDPOINT = 'c2attribution' -OPEN_SOURCE_INDICATORS_ENDPOINT = 'os_indicators' +PASSIVE_DNS_ENDPOINT = "passivedns" +DYNAMIC_DNS_ENDPOINT = "dynamicdns" +PASSIVE_HASH_ENDPOINT = "passivehash" +SINKHOLE_ENDPOINT = "sinkhole" +SSL_CERTIFICATE_ENDPOINT = "ssl_certificate" +DEVICE_GEO_ENDPOINT = "device_geo" +WHOIS_HISTORIC_ENDPOINT = "whois" +WHOIS_CURRENT_ENDPOINT = "whois/v1" +MALWARE_RECORDS_ENDPOINT = "sample" +MALWARE_INFORMATION_ENDPOINT = "sample/information" +C2ATTRIBUTION_ENDPOINT = "c2attribution" +OPEN_SOURCE_INDICATORS_ENDPOINT = "os_indicators" # HYAS API endpoint params -DOMAIN_PARAM = 'domain' -IP_PARAM = 'ip' -IPV4_PARAM = 'ipv4' -IPV6_PARAM = 'ipv6' -EMAIL_PARAM = 'email' -PHONE_PARAM = 'phone' -MD5_PARAM = 'md5' -SHA256_PARAM = 'sha256' -SHA512_PARAM = 'sha512' -HASH_PARAM = 'hash' -SHA1_PARAM = 'sha1' - -HYAS_IP_ENRICHMENT_ENDPOINTS_LIST = [DYNAMIC_DNS_ENDPOINT, PASSIVE_DNS_ENDPOINT, PASSIVE_HASH_ENDPOINT, - SINKHOLE_ENDPOINT, - SSL_CERTIFICATE_ENDPOINT, DEVICE_GEO_ENDPOINT, C2ATTRIBUTION_ENDPOINT, - MALWARE_RECORDS_ENDPOINT, OPEN_SOURCE_INDICATORS_ENDPOINT] -HYAS_DOMAIN_ENRICHMENT_ENDPOINTS_LIST = [PASSIVE_DNS_ENDPOINT, DYNAMIC_DNS_ENDPOINT, WHOIS_HISTORIC_ENDPOINT, - MALWARE_RECORDS_ENDPOINT, WHOIS_CURRENT_ENDPOINT, PASSIVE_HASH_ENDPOINT, - C2ATTRIBUTION_ENDPOINT, SSL_CERTIFICATE_ENDPOINT, - OPEN_SOURCE_INDICATORS_ENDPOINT] -HYAS_EMAIL_ENRICHMENT_ENDPOINTS_LIST = [DYNAMIC_DNS_ENDPOINT, WHOIS_HISTORIC_ENDPOINT, C2ATTRIBUTION_ENDPOINT] +DOMAIN_PARAM = "domain" +IP_PARAM = "ip" +IPV4_PARAM = "ipv4" +IPV6_PARAM = "ipv6" +EMAIL_PARAM = "email" +PHONE_PARAM = "phone" +MD5_PARAM = "md5" +SHA256_PARAM = "sha256" +SHA512_PARAM = "sha512" +HASH_PARAM = "hash" +SHA1_PARAM = "sha1" + +HYAS_IP_ENRICHMENT_ENDPOINTS_LIST = [ + DYNAMIC_DNS_ENDPOINT, + PASSIVE_DNS_ENDPOINT, + PASSIVE_HASH_ENDPOINT, + SINKHOLE_ENDPOINT, + SSL_CERTIFICATE_ENDPOINT, + DEVICE_GEO_ENDPOINT, + C2ATTRIBUTION_ENDPOINT, + MALWARE_RECORDS_ENDPOINT, + OPEN_SOURCE_INDICATORS_ENDPOINT, +] +HYAS_DOMAIN_ENRICHMENT_ENDPOINTS_LIST = [ + PASSIVE_DNS_ENDPOINT, + DYNAMIC_DNS_ENDPOINT, + WHOIS_HISTORIC_ENDPOINT, + MALWARE_RECORDS_ENDPOINT, + WHOIS_CURRENT_ENDPOINT, + PASSIVE_HASH_ENDPOINT, + C2ATTRIBUTION_ENDPOINT, + SSL_CERTIFICATE_ENDPOINT, + OPEN_SOURCE_INDICATORS_ENDPOINT, +] +HYAS_EMAIL_ENRICHMENT_ENDPOINTS_LIST = [ + DYNAMIC_DNS_ENDPOINT, + WHOIS_HISTORIC_ENDPOINT, + C2ATTRIBUTION_ENDPOINT, +] HYAS_PHONE_ENRICHMENT_ENDPOINTS_LIST = [WHOIS_HISTORIC_ENDPOINT] -HYAS_SHA1_ENRICHMENT_ENDPOINTS_LIST = [SSL_CERTIFICATE_ENDPOINT, MALWARE_INFORMATION_ENDPOINT, - OPEN_SOURCE_INDICATORS_ENDPOINT] -HYAS_SHA256_ENRICHMENT_ENDPOINTS_LIST = [C2ATTRIBUTION_ENDPOINT, MALWARE_INFORMATION_ENDPOINT, - OPEN_SOURCE_INDICATORS_ENDPOINT] +HYAS_SHA1_ENRICHMENT_ENDPOINTS_LIST = [ + SSL_CERTIFICATE_ENDPOINT, + MALWARE_INFORMATION_ENDPOINT, + OPEN_SOURCE_INDICATORS_ENDPOINT, +] +HYAS_SHA256_ENRICHMENT_ENDPOINTS_LIST = [ + C2ATTRIBUTION_ENDPOINT, + MALWARE_INFORMATION_ENDPOINT, + OPEN_SOURCE_INDICATORS_ENDPOINT, +] HYAS_SHA512_ENRICHMENT_ENDPOINTS_LIST = [MALWARE_INFORMATION_ENDPOINT] -HYAS_MD5_ENRICHMENT_ENDPOINTS_LIST = [MALWARE_RECORDS_ENDPOINT, MALWARE_INFORMATION_ENDPOINT, - OPEN_SOURCE_INDICATORS_ENDPOINT] +HYAS_MD5_ENRICHMENT_ENDPOINTS_LIST = [ + MALWARE_RECORDS_ENDPOINT, + MALWARE_INFORMATION_ENDPOINT, + OPEN_SOURCE_INDICATORS_ENDPOINT, +] HYAS_OBJECT_NAMES = { DYNAMIC_DNS_ENDPOINT: "Dynamic DNS Information", @@ -144,18 +168,18 @@ WHOIS_CURRENT_ENDPOINT: "Whois Current Related Information", MALWARE_INFORMATION_ENDPOINT: "Malware Sample Information", OPEN_SOURCE_INDICATORS_ENDPOINT: "Open Source Intel for malware, ssl certificates and other indicators Information", - MALWARE_RECORDS_ENDPOINT: "Malware Sample Records Information" + MALWARE_RECORDS_ENDPOINT: "Malware Sample Records Information", } def parse_attribute(comment, feature, value): """Generic Method for parsing the attributes in the object""" attribute = { - 'type': 'text', - 'value': value, - 'comment': comment, - 'distribution': DEFAULT_DISTRIBUTION_SETTING, - 'object_relation': feature + "type": "text", + "value": value, + "comment": comment, + "distribution": DEFAULT_DISTRIBUTION_SETTING, + "object_relation": feature, } return attribute @@ -168,14 +192,14 @@ def misp_object(endpoint, attribute_value): hyas_object.template_id = "1" hyas_object.description = "HYAS INSIGHT " + object_name hyas_object.comment = "HYAS INSIGHT " + object_name + " for " + attribute_value - setattr(hyas_object, 'meta-category', 'network') - description = ( - "An object containing the enriched attribute and " - "related entities from HYAS Insight." - ) + setattr(hyas_object, "meta-category", "network") + description = "An object containing the enriched attribute and related entities from HYAS Insight." hyas_object.from_dict( - **{"meta-category": "misc", "description": description, - "distribution": DEFAULT_DISTRIBUTION_SETTING} + **{ + "meta-category": "misc", + "description": description, + "distribution": DEFAULT_DISTRIBUTION_SETTING, + } ) return hyas_object @@ -187,12 +211,12 @@ def flatten_json(y: Dict) -> Dict[str, Any]: """ out = {} - def flatten(x, name=''): + def flatten(x, name=""): # If the Nested key-value # pair is of dict type if type(x) is dict: for a in x: - flatten(x[a], name + a + '_') + flatten(x[a], name + a + "_") else: out[name[:-1]] = x @@ -219,52 +243,39 @@ def request_body(query_input, query_param, current): """ if current: - return { - "applied_filters": { - query_input: query_param, - "current": True - } - } + return {"applied_filters": {query_input: query_param, "current": True}} else: - return { - "applied_filters": { - query_input: query_param - } - } + return {"applied_filters": {query_input: query_param}} def malware_info_lookup_to_markdown(results: Dict) -> list: - scan_results = results.get('scan_results', []) + scan_results = results.get("scan_results", []) out = [] if scan_results: for res in scan_results: malware_info_data = { - "avscan_score": results.get( - "avscan_score", ''), - "md5": results.get("md5", ''), - 'av_name': res.get( - "av_name", ''), - 'def_time': res.get( - "def_time", ''), - 'threat_found': res.get( - 'threat_found', ''), - 'scan_time': results.get("scan_time", ''), - 'sha1': results.get('sha1', ''), - 'sha256': results.get('sha256', ''), - 'sha512': results.get('sha512', '') + "avscan_score": results.get("avscan_score", ""), + "md5": results.get("md5", ""), + "av_name": res.get("av_name", ""), + "def_time": res.get("def_time", ""), + "threat_found": res.get("threat_found", ""), + "scan_time": results.get("scan_time", ""), + "sha1": results.get("sha1", ""), + "sha256": results.get("sha256", ""), + "sha512": results.get("sha512", ""), } out.append(malware_info_data) else: malware_info_data = { - "avscan_score": results.get("avscan_score", ''), - "md5": results.get("md5", ''), - 'av_name': '', - 'def_time': '', - 'threat_found': '', - 'scan_time': results.get("scan_time", ''), - 'sha1': results.get('sha1', ''), - 'sha256': results.get('sha256', ''), - 'sha512': results.get('sha512', '') + "avscan_score": results.get("avscan_score", ""), + "md5": results.get("md5", ""), + "av_name": "", + "def_time": "", + "threat_found": "", + "scan_time": results.get("scan_time", ""), + "sha1": results.get("sha1", ""), + "sha256": results.get("sha256", ""), + "sha512": results.get("sha512", ""), } out.append(malware_info_data) return out @@ -281,9 +292,7 @@ def get(self, url: str, headers: dict = None, req_body=None) -> requests.Respons """General post method to fetch the response from HYAS Insight.""" response = [] try: - response = self.session.post( - url, headers=headers, json=req_body - ) + response = self.session.post(url, headers=headers, json=req_body) if response: response = response.json() except (ConnectTimeout, ProxyError, InvalidURL) as error: @@ -296,13 +305,13 @@ def hyas_lookup(self, end_point: str, query_input, query_param, current=False) - """Do a lookup call.""" # Building the request if current: - url = f'{WHOIS_CURRENT_BASE_URL}{WHOIS_CURRENT_ENDPOINT}' + url = f"{WHOIS_CURRENT_BASE_URL}{WHOIS_CURRENT_ENDPOINT}" else: - url = f'{HYAS_API_BASE_URL}{end_point}' + url = f"{HYAS_API_BASE_URL}{end_point}" headers = { - 'Content-type': 'application/json', - 'X-API-Key': self.api_key, - 'User-Agent': 'Misp Modules' + "Content-type": "application/json", + "X-API-Key": self.api_key, + "User-Agent": "Misp Modules", } req_body = request_body(query_input, query_param, current) try: @@ -324,366 +333,363 @@ def __init__(self, attribute): self.misp_event.add_attribute(**attribute) self.c2_attribution_data_items = [ - 'actor_ipv4', - 'c2_domain', - 'c2_ip', - 'c2_url', - 'datetime', - 'email', - 'email_domain', - 'referrer_domain', - 'referrer_ipv4', - 'referrer_url', - 'sha256' + "actor_ipv4", + "c2_domain", + "c2_ip", + "c2_url", + "datetime", + "email", + "email_domain", + "referrer_domain", + "referrer_ipv4", + "referrer_url", + "sha256", ] self.c2_attribution_data_items_friendly_names = { - 'actor_ipv4': 'Actor IPv4', - 'c2_domain': 'C2 Domain', - 'c2_ip': 'C2 IP', - 'c2_url': 'C2 URL', - 'datetime': 'DateTime', - 'email': 'Email', - 'email_domain': 'Email Domain', - 'referrer_domain': 'Referrer Domain', - 'referrer_ipv4': 'Referrer IPv4', - 'referrer_url': 'Referrer URL', - 'sha256': 'SHA256' + "actor_ipv4": "Actor IPv4", + "c2_domain": "C2 Domain", + "c2_ip": "C2 IP", + "c2_url": "C2 URL", + "datetime": "DateTime", + "email": "Email", + "email_domain": "Email Domain", + "referrer_domain": "Referrer Domain", + "referrer_ipv4": "Referrer IPv4", + "referrer_url": "Referrer URL", + "sha256": "SHA256", } self.device_geo_data_items = [ - 'datetime', - 'device_user_agent', - 'geo_country_alpha_2', - 'geo_horizontal_accuracy', - 'ipv4', - 'ipv6', - 'latitude', - 'longitude', - 'wifi_bssid' + "datetime", + "device_user_agent", + "geo_country_alpha_2", + "geo_horizontal_accuracy", + "ipv4", + "ipv6", + "latitude", + "longitude", + "wifi_bssid", ] self.device_geo_data_items_friendly_names = { - 'datetime': 'DateTime', - 'device_user_agent': 'Device User Agent', - 'geo_country_alpha_2': 'Alpha-2 Code', - 'geo_horizontal_accuracy': 'GPS Horizontal Accuracy', - 'ipv4': 'IPv4 Address', - 'ipv6': 'IPv6 Address', - 'latitude': 'Latitude', - 'longitude': 'Longitude', - 'wifi_bssid': 'WIFI BSSID' + "datetime": "DateTime", + "device_user_agent": "Device User Agent", + "geo_country_alpha_2": "Alpha-2 Code", + "geo_horizontal_accuracy": "GPS Horizontal Accuracy", + "ipv4": "IPv4 Address", + "ipv6": "IPv6 Address", + "latitude": "Latitude", + "longitude": "Longitude", + "wifi_bssid": "WIFI BSSID", } self.dynamic_dns_data_items = [ - 'a_record', - 'account', - 'created', - 'created_ip', - 'domain', - 'domain_creator_ip', - 'email', + "a_record", + "account", + "created", + "created_ip", + "domain", + "domain_creator_ip", + "email", ] self.dynamic_dns_data_items_friendly_names = { - 'a_record': 'A Record', - 'account': 'Account Holder', - 'created': 'Created Date', - 'created_ip': 'Account Holder IP Address', - 'domain': 'Domain', - 'domain_creator_ip': 'Domain Creator IP Address', - 'email': 'Email Address', + "a_record": "A Record", + "account": "Account Holder", + "created": "Created Date", + "created_ip": "Account Holder IP Address", + "domain": "Domain", + "domain_creator_ip": "Domain Creator IP Address", + "email": "Email Address", } self.os_indicators_data_items = [ - 'context', - 'datetime', - 'domain', - 'domain_2tld', - 'first_seen', - 'ipv4', - 'ipv6', - 'last_seen', - 'md5', - 'sha1', - 'sha256', - 'source_name', - 'source_url', - 'url' + "context", + "datetime", + "domain", + "domain_2tld", + "first_seen", + "ipv4", + "ipv6", + "last_seen", + "md5", + "sha1", + "sha256", + "source_name", + "source_url", + "url", ] self.os_indicators_data_items_friendly_names = { - 'context': 'Context', - 'datetime': 'DateTime', - 'domain': 'Domain', - 'domain_2tld': 'Domain 2TLD', - 'first_seen': 'First Seen', - 'ipv4': 'IPv4 Address', - 'ipv6': 'IPv6 Address', - 'last_seen': 'Last Seen', - 'md5': 'MD5', - 'sha1': 'SHA1', - 'sha256': 'SHA256', - 'source_name': 'Source Name', - 'source_url': 'Source URL', - 'url': 'URL' + "context": "Context", + "datetime": "DateTime", + "domain": "Domain", + "domain_2tld": "Domain 2TLD", + "first_seen": "First Seen", + "ipv4": "IPv4 Address", + "ipv6": "IPv6 Address", + "last_seen": "Last Seen", + "md5": "MD5", + "sha1": "SHA1", + "sha256": "SHA256", + "source_name": "Source Name", + "source_url": "Source URL", + "url": "URL", } self.passive_dns_data_items = [ - 'cert_name', - 'count', - 'domain', - 'first_seen', - 'ip_geo_city_name', - 'ip_geo_country_iso_code', - 'ip_geo_country_name', - 'ip_geo_location_latitude', - 'ip_geo_location_longitude', - 'ip_geo_postal_code', - 'ip_ip', - 'ip_isp_autonomous_system_number', - 'ip_isp_autonomous_system_organization', - 'ip_isp_ip_address', - 'ip_isp_isp', - 'ip_isp_organization', - 'ipv4', - 'ipv6', - 'last_seen' + "cert_name", + "count", + "domain", + "first_seen", + "ip_geo_city_name", + "ip_geo_country_iso_code", + "ip_geo_country_name", + "ip_geo_location_latitude", + "ip_geo_location_longitude", + "ip_geo_postal_code", + "ip_ip", + "ip_isp_autonomous_system_number", + "ip_isp_autonomous_system_organization", + "ip_isp_ip_address", + "ip_isp_isp", + "ip_isp_organization", + "ipv4", + "ipv6", + "last_seen", ] self.passive_dns_data_items_friendly_names = { - 'cert_name': 'Certificate Provider Name', - 'count': 'Passive DNS Count', - 'domain': 'Domain', - 'first_seen': 'First Seen', - 'ip_geo_city_name': 'IP Organization City', - 'ip_geo_country_iso_code': 'IP Organization Country ISO Code', - 'ip_geo_country_name': 'IP Organization Country Name', - 'ip_geo_location_latitude': 'IP Organization Latitude', - 'ip_geo_location_longitude': 'IP Organization Longitude', - 'ip_geo_postal_code': 'IP Organization Postal Code', - 'ip_ip': 'IP Address', - 'ip_isp_autonomous_system_number': 'ASN IP', - 'ip_isp_autonomous_system_organization': 'ASO IP', - 'ip_isp_ip_address': 'IP Address', - 'ip_isp_isp': 'ISP', - 'ip_isp_organization': 'ISP Organization', - 'ipv4': 'IPv4 Address', - 'ipv6': 'IPv6 Address', - 'last_seen': 'Last Seen' + "cert_name": "Certificate Provider Name", + "count": "Passive DNS Count", + "domain": "Domain", + "first_seen": "First Seen", + "ip_geo_city_name": "IP Organization City", + "ip_geo_country_iso_code": "IP Organization Country ISO Code", + "ip_geo_country_name": "IP Organization Country Name", + "ip_geo_location_latitude": "IP Organization Latitude", + "ip_geo_location_longitude": "IP Organization Longitude", + "ip_geo_postal_code": "IP Organization Postal Code", + "ip_ip": "IP Address", + "ip_isp_autonomous_system_number": "ASN IP", + "ip_isp_autonomous_system_organization": "ASO IP", + "ip_isp_ip_address": "IP Address", + "ip_isp_isp": "ISP", + "ip_isp_organization": "ISP Organization", + "ipv4": "IPv4 Address", + "ipv6": "IPv6 Address", + "last_seen": "Last Seen", } - self.passive_hash_data_items = [ - 'domain', - 'md5_count' - ] + self.passive_hash_data_items = ["domain", "md5_count"] self.passive_hash_data_items_friendly_names = { - 'domain': 'Domain', - 'md5_count': 'Passive DNS Count' + "domain": "Domain", + "md5_count": "Passive DNS Count", } self.malware_records_data_items = [ - 'datetime', - 'domain', - 'ipv4', - 'ipv6', - 'md5', - 'sha1', - 'sha256' + "datetime", + "domain", + "ipv4", + "ipv6", + "md5", + "sha1", + "sha256", ] self.malware_records_data_items_friendly_names = { - 'datetime': 'DateTime', - 'domain': 'Domain', - 'ipv4': 'IPv4 Address', - 'ipv6': 'IPv6 Address', - 'md5': 'MD5', - 'sha1': 'SHA1', - 'sha256': 'SHA256' + "datetime": "DateTime", + "domain": "Domain", + "ipv4": "IPv4 Address", + "ipv6": "IPv6 Address", + "md5": "MD5", + "sha1": "SHA1", + "sha256": "SHA256", } self.malware_information_data_items = [ - 'avscan_score', - 'md5', - 'av_name', - 'def_time', - 'threat_found', - 'scan_time', - 'sha1', - 'sha256', - 'sha512' + "avscan_score", + "md5", + "av_name", + "def_time", + "threat_found", + "scan_time", + "sha1", + "sha256", + "sha512", ] self.malware_information_data_items_friendly_names = { - 'avscan_score': 'AV Scan Score', - 'md5': 'MD5', - 'av_name': 'AV Name', - 'def_time': 'AV DateTime', - 'threat_found': 'Source', - 'scan_time': 'Scan DateTime', - 'sha1': 'SHA1', - 'sha256': 'SHA256', - 'sha512': 'SHA512' + "avscan_score": "AV Scan Score", + "md5": "MD5", + "av_name": "AV Name", + "def_time": "AV DateTime", + "threat_found": "Source", + "scan_time": "Scan DateTime", + "sha1": "SHA1", + "sha256": "SHA256", + "sha512": "SHA512", } self.sinkhole_data_items = [ - 'count', - 'country_name', - 'country_code', - 'data_port', - 'datetime', - 'ipv4', - 'last_seen', - 'organization_name', - 'sink_source' + "count", + "country_name", + "country_code", + "data_port", + "datetime", + "ipv4", + "last_seen", + "organization_name", + "sink_source", ] self.sinkhole_data_items_friendly_names = { - 'count': 'Sinkhole Count', - 'country_name': 'IP Address Country', - 'country_code': 'IP Address Country Code', - 'data_port': 'Data Port', - 'datetime': 'First Seen', - 'ipv4': 'IP Address', - 'last_seen': 'Last Seen', - 'organization_name': 'ISP Organization', - 'sink_source': 'Sink Source IP' + "count": "Sinkhole Count", + "country_name": "IP Address Country", + "country_code": "IP Address Country Code", + "data_port": "Data Port", + "datetime": "First Seen", + "ipv4": "IP Address", + "last_seen": "Last Seen", + "organization_name": "ISP Organization", + "sink_source": "Sink Source IP", } self.ssl_certificate_data_items = [ - 'ip', - 'ssl_cert_cert_key', - 'ssl_cert_expire_date', - 'ssl_cert_issue_date', - 'ssl_cert_issuer_commonName', - 'ssl_cert_issuer_countryName', - 'ssl_cert_issuer_localityName', - 'ssl_cert_issuer_organizationName', - 'ssl_cert_issuer_organizationalUnitName', - 'ssl_cert_issuer_stateOrProvinceName', - 'ssl_cert_md5', - 'ssl_cert_serial_number', - 'ssl_cert_sha1', - 'ssl_cert_sha_256', - 'ssl_cert_sig_algo', - 'ssl_cert_ssl_version', - 'ssl_cert_subject_commonName', - 'ssl_cert_subject_countryName', - 'ssl_cert_subject_localityName', - 'ssl_cert_subject_organizationName', - 'ssl_cert_subject_organizationalUnitName', - 'ssl_cert_timestamp' + "ip", + "ssl_cert_cert_key", + "ssl_cert_expire_date", + "ssl_cert_issue_date", + "ssl_cert_issuer_commonName", + "ssl_cert_issuer_countryName", + "ssl_cert_issuer_localityName", + "ssl_cert_issuer_organizationName", + "ssl_cert_issuer_organizationalUnitName", + "ssl_cert_issuer_stateOrProvinceName", + "ssl_cert_md5", + "ssl_cert_serial_number", + "ssl_cert_sha1", + "ssl_cert_sha_256", + "ssl_cert_sig_algo", + "ssl_cert_ssl_version", + "ssl_cert_subject_commonName", + "ssl_cert_subject_countryName", + "ssl_cert_subject_localityName", + "ssl_cert_subject_organizationName", + "ssl_cert_subject_organizationalUnitName", + "ssl_cert_timestamp", ] self.ssl_certificate_data_items_friendly_names = { - 'ip': 'IP Address', - 'ssl_cert_cert_key': 'Certificate Key', - 'ssl_cert_expire_date': 'Certificate Expiration Date', - 'ssl_cert_issue_date': 'Certificate Issue Date', - 'ssl_cert_issuer_commonName': 'Issuer Common Name', - 'ssl_cert_issuer_countryName': 'Issuer Country Name', - 'ssl_cert_issuer_localityName': 'Issuer City Name', - 'ssl_cert_issuer_organizationName': 'Issuer Organization Name', - 'ssl_cert_issuer_organizationalUnitName': 'Issuer Organization Unit Name', - 'ssl_cert_issuer_stateOrProvinceName': 'Issuer State or Province Name', - 'ssl_cert_md5': 'Certificate MD5', - 'ssl_cert_serial_number': 'Certificate Serial Number', - 'ssl_cert_sha1': 'Certificate SHA1', - 'ssl_cert_sha_256': 'Certificate SHA256', - 'ssl_cert_sig_algo': 'Certificate Signature Algorithm', - 'ssl_cert_ssl_version': 'SSL Version', - 'ssl_cert_subject_commonName': 'Reciever Subject Name', - 'ssl_cert_subject_countryName': 'Receiver Country Name', - 'ssl_cert_subject_localityName': 'Receiver City Name', - 'ssl_cert_subject_organizationName': 'Receiver Organization Name', - 'ssl_cert_subject_organizationalUnitName': 'Receiver Organization Unit Name', - 'ssl_cert_timestamp': 'Certificate DateTime' + "ip": "IP Address", + "ssl_cert_cert_key": "Certificate Key", + "ssl_cert_expire_date": "Certificate Expiration Date", + "ssl_cert_issue_date": "Certificate Issue Date", + "ssl_cert_issuer_commonName": "Issuer Common Name", + "ssl_cert_issuer_countryName": "Issuer Country Name", + "ssl_cert_issuer_localityName": "Issuer City Name", + "ssl_cert_issuer_organizationName": "Issuer Organization Name", + "ssl_cert_issuer_organizationalUnitName": "Issuer Organization Unit Name", + "ssl_cert_issuer_stateOrProvinceName": "Issuer State or Province Name", + "ssl_cert_md5": "Certificate MD5", + "ssl_cert_serial_number": "Certificate Serial Number", + "ssl_cert_sha1": "Certificate SHA1", + "ssl_cert_sha_256": "Certificate SHA256", + "ssl_cert_sig_algo": "Certificate Signature Algorithm", + "ssl_cert_ssl_version": "SSL Version", + "ssl_cert_subject_commonName": "Reciever Subject Name", + "ssl_cert_subject_countryName": "Receiver Country Name", + "ssl_cert_subject_localityName": "Receiver City Name", + "ssl_cert_subject_organizationName": "Receiver Organization Name", + "ssl_cert_subject_organizationalUnitName": "Receiver Organization Unit Name", + "ssl_cert_timestamp": "Certificate DateTime", } self.whois_historic_data_items = [ - 'abuse_emails', - 'address', - 'city', - 'country', - 'datetime', - 'domain', - 'domain_2tld', - 'domain_created_datetime', - 'domain_expires_datetime', - 'domain_updated_datetime', - 'email', - 'idn_name', - 'name', - 'nameserver', - 'organization', - 'phone', - 'privacy_punch', - 'registrar' + "abuse_emails", + "address", + "city", + "country", + "datetime", + "domain", + "domain_2tld", + "domain_created_datetime", + "domain_expires_datetime", + "domain_updated_datetime", + "email", + "idn_name", + "name", + "nameserver", + "organization", + "phone", + "privacy_punch", + "registrar", ] self.whois_historic_data_items_friendly_names = { - 'abuse_emails': 'Abuse Emails', - 'address': 'Address', - 'city': 'City', - 'country': 'Country', - 'datetime': 'Datetime', - 'domain': 'Domain', - 'domain_2tld': 'Domain 2tld', - 'domain_created_datetime': 'Domain Created Time', - 'domain_expires_datetime': 'Domain Expires Time', - 'domain_updated_datetime': 'Domain Updated Time', - 'email': 'Email Address', - 'idn_name': 'IDN Name', - 'name': 'Name', - 'nameserver': 'Nameserver', - 'organization': 'Organization', - 'phone': 'Phone Info', - 'privacy_punch': 'Privacy Punch', - 'registrar': 'Registrar' + "abuse_emails": "Abuse Emails", + "address": "Address", + "city": "City", + "country": "Country", + "datetime": "Datetime", + "domain": "Domain", + "domain_2tld": "Domain 2tld", + "domain_created_datetime": "Domain Created Time", + "domain_expires_datetime": "Domain Expires Time", + "domain_updated_datetime": "Domain Updated Time", + "email": "Email Address", + "idn_name": "IDN Name", + "name": "Name", + "nameserver": "Nameserver", + "organization": "Organization", + "phone": "Phone Info", + "privacy_punch": "Privacy Punch", + "registrar": "Registrar", } self.whois_current_data_items = [ - 'abuse_emails', - 'address', - 'city', - 'country', - 'datetime', - 'domain', - 'domain_2tld', - 'domain_created_datetime', - 'domain_expires_datetime', - 'domain_updated_datetime', - 'email', - 'idn_name', - 'name', - 'nameserver', - 'organization', - 'phone', - 'privacy_punch', - 'registrar', - 'state' + "abuse_emails", + "address", + "city", + "country", + "datetime", + "domain", + "domain_2tld", + "domain_created_datetime", + "domain_expires_datetime", + "domain_updated_datetime", + "email", + "idn_name", + "name", + "nameserver", + "organization", + "phone", + "privacy_punch", + "registrar", + "state", ] self.whois_current_data_items_friendly_names = { - 'abuse_emails': 'Abuse Emails', - 'address': 'Address', - 'city': 'City', - 'country': 'Country', - 'datetime': 'Datetime', - 'domain': 'Domain', - 'domain_2tld': 'Domain 2tld', - 'domain_created_datetime': 'Domain Created Time', - 'domain_expires_datetime': 'Domain Expires Time', - 'domain_updated_datetime': 'Domain Updated Time', - 'email': 'Email Address', - 'idn_name': 'IDN Name', - 'name': 'Name', - 'nameserver': 'Nameserver', - 'organization': 'Organization', - 'phone': 'Phone', - 'privacy_punch': 'Privacy Punch', - 'registrar': 'Registrar', - 'state': 'State' + "abuse_emails": "Abuse Emails", + "address": "Address", + "city": "City", + "country": "Country", + "datetime": "Datetime", + "domain": "Domain", + "domain_2tld": "Domain 2tld", + "domain_created_datetime": "Domain Created Time", + "domain_expires_datetime": "Domain Expires Time", + "domain_updated_datetime": "Domain Updated Time", + "email": "Email Address", + "idn_name": "IDN Name", + "name": "Name", + "nameserver": "Nameserver", + "organization": "Organization", + "phone": "Phone", + "privacy_punch": "Privacy Punch", + "registrar": "Registrar", + "state": "State", } def create_misp_attributes_and_objects(self, response, endpoint, attribute_value): @@ -733,45 +739,44 @@ def create_misp_attributes_and_objects(self, response, endpoint, attribute_value if data_item in data_items: data_item_text = data_items_friendly_names[data_item] data_item_value = str(result[data_item]) - hyas_object.add_attribute( - **parse_attribute(hyas_object.comment, data_item_text, data_item_value)) - hyas_object.add_reference(self.attribute['uuid'], 'related-to') + hyas_object.add_attribute(**parse_attribute(hyas_object.comment, data_item_text, data_item_value)) + hyas_object.add_reference(self.attribute["uuid"], "related-to") self.misp_event.add_object(hyas_object) def get_results(self): """returns the dictionary object to MISP Instance""" event = json.loads(self.misp_event.to_json()) - results = {key: event[key] for key in ('Attribute', 'Object')} - return {'results': results} + results = {key: event[key] for key in ("Attribute", "Object")} + return {"results": results} def handler(q=False): """The function which accepts a JSON document to expand the values and return a dictionary of the expanded - values. """ + values.""" if q is False: return False request = json.loads(q) # check if the apikey is provided - if not request.get('config') or not request['config'].get('apikey'): - misperrors['error'] = 'HYAS Insight apikey is missing' + if not request.get("config") or not request["config"].get("apikey"): + misperrors["error"] = "HYAS Insight apikey is missing" return misperrors - apikey = request['config'].get('apikey') + apikey = request["config"].get("apikey") # check attribute is added to the event - if not request.get('attribute') or not check_input_attribute(request['attribute']): - return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} + if not request.get("attribute") or not check_input_attribute(request["attribute"]): + return {"error": f"{standard_error_message}, which should contain at least a type, a value and an uuid."} - attribute = request['attribute'] - attribute_type = attribute['type'] - attribute_value = attribute['value'] + attribute = request["attribute"] + attribute_type = attribute["type"] + attribute_value = attribute["value"] # check if the attribute type is supported by IPQualityScore - if attribute_type not in mispattributes['input']: - return {'error': 'Unsupported attributes type for HYAS Insight Enrichment'} + if attribute_type not in mispattributes["input"]: + return {"error": "Unsupported attributes type for HYAS Insight Enrichment"} request_handler = RequestHandler(apikey) parser = HyasInsightParser(attribute) has_results = False if attribute_type in ip_query_input_type: - ip_param = '' + ip_param = "" for endpoint in HYAS_IP_ENRICHMENT_ENDPOINTS_LIST: if endpoint == DEVICE_GEO_ENDPOINT: if re.match(IPV4_REGEX, attribute_value): @@ -788,7 +793,7 @@ def handler(q=False): ip_param = IP_PARAM enrich_response = request_handler.hyas_lookup(endpoint, ip_param, attribute_value) if endpoint == SSL_CERTIFICATE_ENDPOINT: - enrich_response = enrich_response.get('ssl_certs') + enrich_response = enrich_response.get("ssl_certs") if enrich_response: has_results = True parser.create_misp_attributes_and_objects(enrich_response, endpoint, attribute_value) @@ -797,9 +802,13 @@ def handler(q=False): if not endpoint == WHOIS_CURRENT_ENDPOINT: enrich_response = request_handler.hyas_lookup(endpoint, DOMAIN_PARAM, attribute_value) else: - enrich_response = request_handler.hyas_lookup(endpoint, DOMAIN_PARAM, attribute_value, - endpoint == WHOIS_CURRENT_ENDPOINT) - enrich_response = enrich_response.get('items') + enrich_response = request_handler.hyas_lookup( + endpoint, + DOMAIN_PARAM, + attribute_value, + endpoint == WHOIS_CURRENT_ENDPOINT, + ) + enrich_response = enrich_response.get("items") if enrich_response: has_results = True parser.create_misp_attributes_and_objects(enrich_response, endpoint, attribute_value) @@ -852,7 +861,7 @@ def handler(q=False): enrich_response = malware_info_lookup_to_markdown(enrich_response) parser.create_misp_attributes_and_objects(enrich_response, endpoint, attribute_value) elif attribute_type in sha512_query_input_type: - sha512_param = '' + sha512_param = "" for endpoint in HYAS_SHA512_ENRICHMENT_ENDPOINTS_LIST: if endpoint == MALWARE_INFORMATION_ENDPOINT: sha512_param = HASH_PARAM @@ -866,7 +875,7 @@ def handler(q=False): if has_results: return parser.get_results() else: - return {'error': 'No records found in HYAS Insight for the provided attribute.'} + return {"error": "No records found in HYAS Insight for the provided attribute."} def introspection(): @@ -876,6 +885,6 @@ def introspection(): def version(): """The function that returns a dict with the version and the associated meta-data including potential - configurations required of the module. """ - moduleinfo['config'] = moduleconfig + configurations required of the module.""" + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/intel471.py b/misp_modules/modules/expansion/intel471.py index a8127c7af..bf0ab4ce9 100755 --- a/misp_modules/modules/expansion/intel471.py +++ b/misp_modules/modules/expansion/intel471.py @@ -1,40 +1,66 @@ import json + from pyintel471 import PyIntel471 -misperrors = {'error': 'Error'} -mispattributes = {'input': ['hostname', 'domain', 'url', 'ip-src', 'ip-dst', 'email-src', - 'email-dst', 'target-email', 'whois-registrant-email', - 'whois-registrant-name', 'md5', 'sha1', 'sha256'], 'output': ['freetext']} +misperrors = {"error": "Error"} +mispattributes = { + "input": [ + "hostname", + "domain", + "url", + "ip-src", + "ip-dst", + "email-src", + "email-dst", + "target-email", + "whois-registrant-email", + "whois-registrant-name", + "md5", + "sha1", + "sha256", + ], + "output": ["freetext"], +} moduleinfo = { - 'version': '0.1', - 'author': 'Raphaël Vinot', - 'description': 'Module to access Intel 471', - 'module-type': ['hover', 'expansion'], - 'name': 'Intel471 Lookup', - 'logo': 'intel471.png', - 'requirements': ['The intel471 python library'], - 'features': 'The module uses the Intel471 python library to query the Intel471 API with the value of the input attribute. The result of the query is then returned as freetext so the Freetext import parses it.', - 'references': ['https://public.intel471.com/'], - 'input': 'A MISP attribute whose type is included in the following list:\n- hostname\n- domain\n- url\n- ip-src\n- ip-dst\n- email-src\n- email-dst\n- target-email\n- whois-registrant-email\n- whois-registrant-name\n- md5\n- sha1\n- sha256', - 'output': 'Freetext', - 'descrption': 'An expansion module to query Intel471 in order to get additional information about a domain, ip address, email address, url or hash.', + "version": "0.1", + "author": "Raphaël Vinot", + "description": "Module to access Intel 471", + "module-type": ["hover", "expansion"], + "name": "Intel471 Lookup", + "logo": "intel471.png", + "requirements": ["The intel471 python library"], + "features": ( + "The module uses the Intel471 python library to query the Intel471 API with the value of the input attribute." + " The result of the query is then returned as freetext so the Freetext import parses it." + ), + "references": ["https://public.intel471.com/"], + "input": ( + "A MISP attribute whose type is included in the following list:\n- hostname\n- domain\n- url\n- ip-src\n-" + " ip-dst\n- email-src\n- email-dst\n- target-email\n- whois-registrant-email\n- whois-registrant-name\n- md5\n-" + " sha1\n- sha256" + ), + "output": "Freetext", + "descrption": ( + "An expansion module to query Intel471 in order to get additional information about a domain, ip address, email" + " address, url or hash." + ), } -moduleconfig = ['email', 'authkey'] +moduleconfig = ["email", "authkey"] def cleanup(response): - '''The entries have uids that will be recognised as hashes when they shouldn't''' + """The entries have uids that will be recognised as hashes when they shouldn't""" j = response.json() - if j['iocTotalCount'] == 0: - return 'Nothing has been found.' - for ioc in j['iocs']: - ioc.pop('uid') - if ioc['links']['actorTotalCount'] > 0: - for actor in ioc['links']['actors']: - actor.pop('uid') - if ioc['links']['reportTotalCount'] > 0: - for report in ioc['links']['reports']: - report.pop('uid') + if j["iocTotalCount"] == 0: + return "Nothing has been found." + for ioc in j["iocs"]: + ioc.pop("uid") + if ioc["links"]["actorTotalCount"] > 0: + for actor in ioc["links"]["actors"]: + actor.pop("uid") + if ioc["links"]["reportTotalCount"] > 0: + for report in ioc["links"]["reports"]: + report.pop("uid") return json.dumps(j, indent=2) @@ -42,25 +68,25 @@ def handler(q=False): if q is False: return False request = json.loads(q) - for input_type in mispattributes['input']: + for input_type in mispattributes["input"]: if input_type in request: to_query = request[input_type] break else: - misperrors['error'] = "Unsupported attributes type" + misperrors["error"] = "Unsupported attributes type" return misperrors - if (request.get('config')): - if (request['config'].get('email') is None) or (request['config'].get('authkey') is None): - misperrors['error'] = 'Intel 471 authentication is missing' + if request.get("config"): + if (request["config"].get("email") is None) or (request["config"].get("authkey") is None): + misperrors["error"] = "Intel 471 authentication is missing" return misperrors - intel471 = PyIntel471(email=request['config'].get('email'), authkey=request['config'].get('authkey')) + intel471 = PyIntel471(email=request["config"].get("email"), authkey=request["config"].get("authkey")) ioc_filters = intel471.iocs_filters(ioc=to_query) res = intel471.iocs(filters=ioc_filters) to_return = cleanup(res) - r = {'results': [{'types': mispattributes['output'], 'values': to_return}]} + r = {"results": [{"types": mispattributes["output"], "values": to_return}]} return r @@ -69,5 +95,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/ip2locationio.py b/misp_modules/modules/expansion/ip2locationio.py index 607ff0426..8a711c052 100644 --- a/misp_modules/modules/expansion/ip2locationio.py +++ b/misp_modules/modules/expansion/ip2locationio.py @@ -1,35 +1,44 @@ import json + import requests -from . import check_input_attribute, standard_error_message from pymisp import MISPAttribute, MISPEvent, MISPObject -mispattributes = { - 'input': ['ip-src', 'ip-dst'], - 'format': 'misp_standard' -} +from . import check_input_attribute, standard_error_message + +mispattributes = {"input": ["ip-src", "ip-dst"], "format": "misp_standard"} moduleinfo = { - 'version': 1, - 'author': 'IP2Location.io', - 'description': 'An expansion module to query IP2Location.io to gather more information on a given IP address.', - 'module-type': ['expansion', 'hover'], - 'name': 'IP2Location.io Lookup', - 'logo': 'ip2locationio.png', - 'requirements': ['An IP2Location.io token'], - 'features': 'The module takes an IP address attribute as input and queries the IP2Location.io API. \nFree plan user will get the basic geolocation informaiton, and different subsription plan will get more information on the IP address. \n Refer to [pricing page](https://www.ip2location.io/pricing) for more information on data available for each plan. \n\nMore information on the responses content is available in the [documentation](https://www.ip2location.io/ip2location-documentation).', - 'references': ['https://www.ip2location.io/ip2location-documentation'], - 'input': 'IP address attribute.', - 'output': 'Additional information on the IP address, such as geolocation, proxy and so on. Refer to the Response Format section in https://www.ip2location.io/ip2location-documentation to find out the full format of the data returned.', + "version": 1, + "author": "IP2Location.io", + "description": "An expansion module to query IP2Location.io to gather more information on a given IP address.", + "module-type": ["expansion", "hover"], + "name": "IP2Location.io Lookup", + "logo": "ip2locationio.png", + "requirements": ["An IP2Location.io token"], + "features": ( + "The module takes an IP address attribute as input and queries the IP2Location.io API. \nFree plan user will" + " get the basic geolocation informaiton, and different subsription plan will get more information on the IP" + " address. \n Refer to [pricing page](https://www.ip2location.io/pricing) for more information on data" + " available for each plan. \n\nMore information on the responses content is available in the" + " [documentation](https://www.ip2location.io/ip2location-documentation)." + ), + "references": ["https://www.ip2location.io/ip2location-documentation"], + "input": "IP address attribute.", + "output": ( + "Additional information on the IP address, such as geolocation, proxy and so on. Refer to the Response Format" + " section in https://www.ip2location.io/ip2location-documentation to find out the full format of the data" + " returned." + ), } -moduleconfig = ['key'] +moduleconfig = ["key"] _GEOLOCATION_OBJECT_MAPPING = { - 'country_code': 'countrycode', - 'country_name': 'country', - 'region_name': 'region', - 'city_name': 'city', - 'zip_code': 'zipcode', - 'latitude': 'latitude', - 'longitude': 'longitude' + "country_code": "countrycode", + "country_name": "country", + "region_name": "region", + "city_name": "city", + "zip_code": "zipcode", + "latitude": "latitude", + "longitude": "longitude", } @@ -38,28 +47,26 @@ def handler(q=False): if q is False: return False request = json.loads(q) - if not request.get('attribute') or not check_input_attribute(request['attribute']): - return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} - attribute = request['attribute'] - if attribute.get('type') not in mispattributes['input']: - return {'error': 'Wrong input attribute type.'} - if not request.get('config'): - return {'error': 'Missing ip2locationio config.'} - if not request['config'].get('key'): - return {'error': 'Missing ip2locationio API key.'} + if not request.get("attribute") or not check_input_attribute(request["attribute"]): + return {"error": f"{standard_error_message}, which should contain at least a type, a value and an uuid."} + attribute = request["attribute"] + if attribute.get("type") not in mispattributes["input"]: + return {"error": "Wrong input attribute type."} + if not request.get("config"): + return {"error": "Missing ip2locationio config."} + if not request["config"].get("key"): + return {"error": "Missing ip2locationio API key."} # Query ip2location.io - query = requests.get( - f"https://api.ip2location.io/json?key={request['config']['key']}&ip={attribute['value']}" - ) + query = requests.get(f"https://api.ip2location.io/json?key={request['config']['key']}&ip={attribute['value']}") if query.status_code != 200: - return {'error': f'Error while querying ip2location.io - {query.status_code}: {query.reason}'} + return {"error": f"Error while querying ip2location.io - {query.status_code}: {query.reason}"} iplio_result = query.json() # Check if the IP address is not reserved for special use # if ipinfo.get('bogon', False): - if '' in iplio_result and iplio_result[''] == 'RSV': - return {'error': 'The IP address is reserved for special use'} + if "" in iplio_result and iplio_result[""] == "RSV": + return {"error": "The IP address is reserved for special use"} # Initiate the MISP data structures misp_event = MISPEvent() @@ -68,17 +75,15 @@ def handler(q=False): misp_event.add_attribute(**input_attribute) # Parse the geolocation information related to the IP address - geolocation = MISPObject('geolocation') + geolocation = MISPObject("geolocation") for field, relation in _GEOLOCATION_OBJECT_MAPPING.items(): geolocation.add_attribute(relation, iplio_result[field]) - geolocation.add_reference(input_attribute.uuid, 'locates') + geolocation.add_reference(input_attribute.uuid, "locates") misp_event.add_object(geolocation) # Return the results in MISP format event = json.loads(misp_event.to_json()) - return { - 'results': {key: event[key] for key in ('Attribute', 'Object')} - } + return {"results": {key: event[key] for key in ("Attribute", "Object")}} def introspection(): @@ -86,5 +91,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/ipasn.py b/misp_modules/modules/expansion/ipasn.py index ee05af8f4..9cdb32286 100755 --- a/misp_modules/modules/expansion/ipasn.py +++ b/misp_modules/modules/expansion/ipasn.py @@ -1,28 +1,31 @@ # -*- coding: utf-8 -*- import json -from . import check_input_attribute, standard_error_message + from pyipasnhistory import IPASNHistory from pymisp import MISPAttribute, MISPEvent, MISPObject -misperrors = {'error': 'Error'} -mispattributes = {'input': ['ip-src', 'ip-dst', 'ip'], 'format': 'misp_standard'} +from . import check_input_attribute, standard_error_message + +misperrors = {"error": "Error"} +mispattributes = {"input": ["ip-src", "ip-dst", "ip"], "format": "misp_standard"} moduleinfo = { - 'version': '0.3', - 'author': 'Raphaël Vinot', - 'description': 'Module to query an IP ASN history service (https://github.com/D4-project/IPASN-History).', - 'module-type': ['expansion', 'hover'], - 'name': 'IPASN-History Lookup', - 'logo': '', - 'requirements': ['pyipasnhistory: Python library to access IPASN-history instance'], - 'features': 'This module takes an IP address attribute as input and queries the CIRCL IPASN service. The result of the query is the latest asn related to the IP address, that is returned as a MISP object.', - 'references': ['https://github.com/D4-project/IPASN-History'], - 'input': 'An IP address MISP attribute.', - 'output': 'Asn object(s) objects related to the IP address used as input.', + "version": "0.3", + "author": "Raphaël Vinot", + "description": "Module to query an IP ASN history service (https://github.com/D4-project/IPASN-History).", + "module-type": ["expansion", "hover"], + "name": "IPASN-History Lookup", + "logo": "", + "requirements": ["pyipasnhistory: Python library to access IPASN-history instance"], + "features": ( + "This module takes an IP address attribute as input and queries the CIRCL IPASN service. The result of the" + " query is the latest asn related to the IP address, that is returned as a MISP object." + ), + "references": ["https://github.com/D4-project/IPASN-History"], + "input": "An IP address MISP attribute.", + "output": "Asn object(s) objects related to the IP address used as input.", } -moduleconfig = [ - "custom_api" -] +moduleconfig = ["custom_api"] def parse_result(attribute, values): @@ -30,41 +33,41 @@ def parse_result(attribute, values): initial_attribute = MISPAttribute() initial_attribute.from_dict(**attribute) event.add_attribute(**initial_attribute) - mapping = {'asn': ('AS', 'asn'), 'prefix': ('ip-src', 'subnet-announced')} - for last_seen, response in values['response'].items(): - asn = MISPObject('asn') - asn.add_attribute('last-seen', **{'type': 'datetime', 'value': last_seen}) + mapping = {"asn": ("AS", "asn"), "prefix": ("ip-src", "subnet-announced")} + for last_seen, response in values["response"].items(): + asn = MISPObject("asn") + asn.add_attribute("last-seen", **{"type": "datetime", "value": last_seen}) for feature, attribute_fields in mapping.items(): attribute_type, object_relation = attribute_fields - asn.add_attribute(object_relation, **{'type': attribute_type, 'value': response[feature]}) - asn.add_reference(initial_attribute.uuid, 'related-to') + asn.add_attribute(object_relation, **{"type": attribute_type, "value": response[feature]}) + asn.add_reference(initial_attribute.uuid, "related-to") event.add_object(**asn) event = json.loads(event.to_json()) - return {key: event[key] for key in ('Attribute', 'Object')} + return {key: event[key] for key in ("Attribute", "Object")} def handler(q=False): if q is False: return False request = json.loads(q) - if not request.get('attribute') or not check_input_attribute(request['attribute']): - return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} - if request['attribute']['type'] not in mispattributes['input']: - return {'error': 'Unsupported attribute type.'} - if request['attribute']['type'] == 'ip': - request['attribute']['type'] = 'ip-src' + if not request.get("attribute") or not check_input_attribute(request["attribute"]): + return {"error": f"{standard_error_message}, which should contain at least a type, a value and an uuid."} + if request["attribute"]["type"] not in mispattributes["input"]: + return {"error": "Unsupported attribute type."} + if request["attribute"]["type"] == "ip": + request["attribute"]["type"] = "ip-src" - toquery = request['attribute']['value'] + toquery = request["attribute"]["value"] ipasn_url = request["config"].get("custom_api") or "https://ipasnhistory.circl.lu/" - + ipasn = IPASNHistory(root_url=ipasn_url) values = ipasn.query(toquery) if not values: - misperrors['error'] = 'Unable to find the history of this IP' + misperrors["error"] = "Unable to find the history of this IP" return misperrors - return {'results': parse_result(request['attribute'], values)} + return {"results": parse_result(request["attribute"], values)} def introspection(): diff --git a/misp_modules/modules/expansion/ipinfo.py b/misp_modules/modules/expansion/ipinfo.py index 6fb0ca273..52b055e64 100644 --- a/misp_modules/modules/expansion/ipinfo.py +++ b/misp_modules/modules/expansion/ipinfo.py @@ -1,32 +1,43 @@ import json + import requests -from . import check_input_attribute, standard_error_message from pymisp import MISPAttribute, MISPEvent, MISPObject -mispattributes = { - 'input': ['ip-src', 'ip-dst'], - 'format': 'misp_standard' -} +from . import check_input_attribute, standard_error_message + +mispattributes = {"input": ["ip-src", "ip-dst"], "format": "misp_standard"} moduleinfo = { - 'version': 1, - 'author': 'Christian Studer', - 'description': 'An expansion module to query ipinfo.io to gather more information on a given IP address.', - 'module-type': ['expansion', 'hover'], - 'name': 'IPInfo.io Lookup', - 'logo': 'ipinfo.png', - 'requirements': ['An ipinfo.io token'], - 'features': 'The module takes an IP address attribute as input and queries the ipinfo.io API. \nThe geolocation information on the IP address is always returned.\n\nDepending on the subscription plan, the API returns different pieces of information then:\n- With a basic plan (free) you get the AS number and the AS organisation name concatenated in the `org` field.\n- With a paid subscription, the AS information is returned in the `asn` field with additional AS information, and depending on which plan the user has, you can also get information on the privacy method used to protect the IP address, the related domains, or the point of contact related to the IP address in case of an abuse.\n\nMore information on the responses content is available in the [documentation](https://ipinfo.io/developers).', - 'references': ['https://ipinfo.io/developers'], - 'input': 'IP address attribute.', - 'output': 'Additional information on the IP address, like its geolocation, the autonomous system it is included in, and the related domain(s).', + "version": 1, + "author": "Christian Studer", + "description": "An expansion module to query ipinfo.io to gather more information on a given IP address.", + "module-type": ["expansion", "hover"], + "name": "IPInfo.io Lookup", + "logo": "ipinfo.png", + "requirements": ["An ipinfo.io token"], + "features": ( + "The module takes an IP address attribute as input and queries the ipinfo.io API. \nThe geolocation" + " information on the IP address is always returned.\n\nDepending on the subscription plan, the API returns" + " different pieces of information then:\n- With a basic plan (free) you get the AS number and the AS" + " organisation name concatenated in the `org` field.\n- With a paid subscription, the AS information is" + " returned in the `asn` field with additional AS information, and depending on which plan the user has, you can" + " also get information on the privacy method used to protect the IP address, the related domains, or the point" + " of contact related to the IP address in case of an abuse.\n\nMore information on the responses content is" + " available in the [documentation](https://ipinfo.io/developers)." + ), + "references": ["https://ipinfo.io/developers"], + "input": "IP address attribute.", + "output": ( + "Additional information on the IP address, like its geolocation, the autonomous system it is included in, and" + " the related domain(s)." + ), } -moduleconfig = ['token'] +moduleconfig = ["token"] _GEOLOCATION_OBJECT_MAPPING = { - 'city': 'city', - 'postal': 'zipcode', - 'region': 'region', - 'country': 'countrycode' + "city": "city", + "postal": "zipcode", + "region": "region", + "country": "countrycode", } @@ -35,27 +46,25 @@ def handler(q=False): if q is False: return False request = json.loads(q) - if not request.get('attribute') or not check_input_attribute(request['attribute']): - return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} - attribute = request['attribute'] - if attribute.get('type') not in mispattributes['input']: - return {'error': 'Wrong input attribute type.'} - if not request.get('config'): - return {'error': 'Missing ipinfo config.'} - if not request['config'].get('token'): - return {'error': 'Missing ipinfo token.'} + if not request.get("attribute") or not check_input_attribute(request["attribute"]): + return {"error": f"{standard_error_message}, which should contain at least a type, a value and an uuid."} + attribute = request["attribute"] + if attribute.get("type") not in mispattributes["input"]: + return {"error": "Wrong input attribute type."} + if not request.get("config"): + return {"error": "Missing ipinfo config."} + if not request["config"].get("token"): + return {"error": "Missing ipinfo token."} # Query ipinfo.io - query = requests.get( - f"https://ipinfo.io/{attribute['value']}/json?token={request['config']['token']}" - ) + query = requests.get(f"https://ipinfo.io/{attribute['value']}/json?token={request['config']['token']}") if query.status_code != 200: - return {'error': f'Error while querying ipinfo.io - {query.status_code}: {query.reason}'} + return {"error": f"Error while querying ipinfo.io - {query.status_code}: {query.reason}"} ipinfo = query.json() # Check if the IP address is not reserved for special use - if ipinfo.get('bogon', False): - return {'error': 'The IP address is reserved for special use'} + if ipinfo.get("bogon", False): + return {"error": "The IP address is reserved for special use"} # Initiate the MISP data structures misp_event = MISPEvent() @@ -64,43 +73,40 @@ def handler(q=False): misp_event.add_attribute(**input_attribute) # Parse the geolocation information related to the IP address - geolocation = MISPObject('geolocation') + geolocation = MISPObject("geolocation") for field, relation in _GEOLOCATION_OBJECT_MAPPING.items(): geolocation.add_attribute(relation, ipinfo[field]) - for relation, value in zip(('latitude', 'longitude'), ipinfo['loc'].split(',')): + for relation, value in zip(("latitude", "longitude"), ipinfo["loc"].split(",")): geolocation.add_attribute(relation, value) - geolocation.add_reference(input_attribute.uuid, 'locates') + geolocation.add_reference(input_attribute.uuid, "locates") misp_event.add_object(geolocation) # Parse the domain information - domain_ip = misp_event.add_object(name='domain-ip') - for feature in ('hostname', 'ip'): + domain_ip = misp_event.add_object(name="domain-ip") + for feature in ("hostname", "ip"): domain_ip.add_attribute(feature, ipinfo[feature]) - domain_ip.add_reference(input_attribute.uuid, 'resolves') - if ipinfo.get('domain') is not None: - for domain in ipinfo['domain']['domains']: - domain_ip.add_attribute('domain', domain) + domain_ip.add_reference(input_attribute.uuid, "resolves") + if ipinfo.get("domain") is not None: + for domain in ipinfo["domain"]["domains"]: + domain_ip.add_attribute("domain", domain) # Parse the AS information - asn = MISPObject('asn') - asn.add_reference(input_attribute.uuid, 'includes') - if ipinfo.get('asn') is not None: - asn_info = ipinfo['asn'] - asn.add_attribute('asn', asn_info['asn']) - asn.add_attribute('description', asn_info['name']) + asn = MISPObject("asn") + asn.add_reference(input_attribute.uuid, "includes") + if ipinfo.get("asn") is not None: + asn_info = ipinfo["asn"] + asn.add_attribute("asn", asn_info["asn"]) + asn.add_attribute("description", asn_info["name"]) misp_event.add_object(asn) - elif ipinfo.get('org'): - as_value, *description = ipinfo['org'].split(' ') - asn.add_attribute('asn', as_value) - asn.add_attribute('description', ' '.join(description)) + elif ipinfo.get("org"): + as_value, *description = ipinfo["org"].split(" ") + asn.add_attribute("asn", as_value) + asn.add_attribute("description", " ".join(description)) misp_event.add_object(asn) - # Return the results in MISP format event = json.loads(misp_event.to_json()) - return { - 'results': {key: event[key] for key in ('Attribute', 'Object')} - } + return {"results": {key: event[key] for key in ("Attribute", "Object")}} def introspection(): @@ -108,5 +114,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/ipqs_fraud_and_risk_scoring.py b/misp_modules/modules/expansion/ipqs_fraud_and_risk_scoring.py index 680637751..7d2449ec9 100644 --- a/misp_modules/modules/expansion/ipqs_fraud_and_risk_scoring.py +++ b/misp_modules/modules/expansion/ipqs_fraud_and_risk_scoring.py @@ -1,67 +1,63 @@ import json import logging + import requests -from requests.exceptions import ( - HTTPError, - ProxyError, - InvalidURL, - ConnectTimeout -) +from pymisp import Distribution, MISPAttribute, MISPEvent, MISPObject, MISPTag +from requests.exceptions import ConnectTimeout, HTTPError, InvalidURL, ProxyError + from . import check_input_attribute, standard_error_message -from pymisp import MISPEvent, MISPAttribute, MISPObject, MISPTag, Distribution -ip_query_input_type = [ - 'ip-src', - 'ip-dst' -] -url_query_input_type = [ - 'hostname', - 'domain', - 'url', - 'uri' -] +ip_query_input_type = ["ip-src", "ip-dst"] +url_query_input_type = ["hostname", "domain", "url", "uri"] email_query_input_type = [ - 'email', - 'email-src', - 'email-dst', - 'target-email', - 'whois-registrant-email' -] -phone_query_input_type = [ - 'phone-number', - 'whois-registrant-phone' + "email", + "email-src", + "email-dst", + "target-email", + "whois-registrant-email", ] +phone_query_input_type = ["phone-number", "whois-registrant-phone"] -misperrors = { - 'error': 'Error' -} +misperrors = {"error": "Error"} mispattributes = { - 'input': ip_query_input_type + url_query_input_type + email_query_input_type + phone_query_input_type, - 'format': 'misp_standard' + "input": ip_query_input_type + url_query_input_type + email_query_input_type + phone_query_input_type, + "format": "misp_standard", } moduleinfo = { - 'version': '0.1', - 'author': 'David Mackler', - 'description': 'IPQualityScore MISP Expansion Module for IP reputation, Email Validation, Phone Number Validation, Malicious Domain and Malicious URL Scanner.', - 'module-type': ['expansion', 'hover'], - 'name': 'IPQualityScore Lookup', - 'logo': 'ipqualityscore.png', - 'requirements': ['A IPQualityScore API Key.'], - 'features': 'This Module takes the IP Address, Domain, URL, Email and Phone Number MISP Attributes as input to query the IPQualityScore API.\n The results of the IPQualityScore API are than returned as IPQS Fraud and Risk Scoring Object. \n The object contains a copy of the enriched attribute with added tags presenting the verdict based on fraud score,risk score and other attributes from IPQualityScore.', - 'references': ['https://www.ipqualityscore.com/'], - 'input': 'A MISP attribute of type IP Address(ip-src, ip-dst), Domain(hostname, domain), URL(url, uri), Email Address(email, email-src, email-dst, target-email, whois-registrant-email) and Phone Number(phone-number, whois-registrant-phone).', - 'output': 'IPQualityScore object, resulting from the query on the IPQualityScore API.', + "version": "0.1", + "author": "David Mackler", + "description": ( + "IPQualityScore MISP Expansion Module for IP reputation, Email Validation, Phone Number Validation, Malicious" + " Domain and Malicious URL Scanner." + ), + "module-type": ["expansion", "hover"], + "name": "IPQualityScore Lookup", + "logo": "ipqualityscore.png", + "requirements": ["A IPQualityScore API Key."], + "features": ( + "This Module takes the IP Address, Domain, URL, Email and Phone Number MISP Attributes as input to query the" + " IPQualityScore API.\n The results of the IPQualityScore API are than returned as IPQS Fraud and Risk Scoring" + " Object. \n The object contains a copy of the enriched attribute with added tags presenting the verdict based" + " on fraud score,risk score and other attributes from IPQualityScore." + ), + "references": ["https://www.ipqualityscore.com/"], + "input": ( + "A MISP attribute of type IP Address(ip-src, ip-dst), Domain(hostname, domain), URL(url, uri), Email" + " Address(email, email-src, email-dst, target-email, whois-registrant-email) and Phone Number(phone-number," + " whois-registrant-phone)." + ), + "output": "IPQualityScore object, resulting from the query on the IPQualityScore API.", } -moduleconfig = ['apikey'] +moduleconfig = ["apikey"] -logger = logging.getLogger('ipqualityscore') +logger = logging.getLogger("ipqualityscore") logger.setLevel(logging.DEBUG) -BASE_URL = 'https://ipqualityscore.com/api/json' +BASE_URL = "https://ipqualityscore.com/api/json" DEFAULT_DISTRIBUTION_SETTING = Distribution.your_organisation_only.value -IP_ENRICH = 'ip' -URL_ENRICH = 'url' -EMAIL_ENRICH = 'email' -PHONE_ENRICH = 'phone' +IP_ENRICH = "ip" +URL_ENRICH = "url" +EMAIL_ENRICH = "email" +PHONE_ENRICH = "phone" class RequestHandler: @@ -74,9 +70,7 @@ def __init__(self, apikey): def get(self, url: str, headers: dict = None, params: dict = None) -> requests.Response: """General get method to fetch the response from IPQualityScore.""" try: - response = self.session.get( - url, headers=headers, params=params - ).json() + response = self.session.get(url, headers=headers, params=params).json() if str(response["success"]) != "True": msg = response["message"] logger.error(f"Error: {msg}") @@ -106,11 +100,11 @@ def ipqs_lookup(self, reputation_type: str, ioc: str) -> requests.Response: def parse_attribute(comment, feature, value): """Generic Method for parsing the attributes in the object""" attribute = { - 'type': 'text', - 'value': value, - 'comment': comment, - 'distribution': DEFAULT_DISTRIBUTION_SETTING, - 'object_relation': feature + "type": "text", + "value": value, + "comment": comment, + "distribution": DEFAULT_DISTRIBUTION_SETTING, + "object_relation": feature, } return attribute @@ -136,214 +130,215 @@ def __init__(self, attribute): self.attribute = attribute self.misp_event = MISPEvent() self.misp_event.add_attribute(**attribute) - self.ipqs_object = MISPObject('IPQS Fraud and Risk Scoring Object') + self.ipqs_object = MISPObject("IPQS Fraud and Risk Scoring Object") self.ipqs_object.template_uuid = "57d066e6-6d66-42a7-a1ad-e075e39b2b5e" self.ipqs_object.template_id = "1" self.ipqs_object.description = "IPQS Fraud and Risk Scoring Data" - setattr(self.ipqs_object, 'meta-category', 'network') - description = ( - "An object containing the enriched attribute and " - "related entities from IPQualityScore." - ) + setattr(self.ipqs_object, "meta-category", "network") + description = "An object containing the enriched attribute and related entities from IPQualityScore." self.ipqs_object.from_dict( - **{"meta-category": "misc", "description": description, "distribution": DEFAULT_DISTRIBUTION_SETTING} + **{ + "meta-category": "misc", + "description": description, + "distribution": DEFAULT_DISTRIBUTION_SETTING, + } ) temp_attr = MISPAttribute() temp_attr.from_dict(**attribute) self.enriched_attribute = MISPAttribute() self.enriched_attribute.from_dict( - **{"value": temp_attr.value, "type": temp_attr.type, "distribution": DEFAULT_DISTRIBUTION_SETTING} + **{ + "value": temp_attr.value, + "type": temp_attr.type, + "distribution": DEFAULT_DISTRIBUTION_SETTING, + } ) self.ipqs_object.distribution = DEFAULT_DISTRIBUTION_SETTING self.ip_data_items = [ - 'fraud_score', - 'country_code', - 'region', - 'city', - 'zip_code', - 'ISP', - 'ASN', - 'organization', - 'is_crawler', - 'timezone', - 'mobile', - 'host', - 'proxy', - 'vpn', - 'tor', - 'active_vpn', - 'active_tor', - 'recent_abuse', - 'bot_status', - 'connection_type', - 'abuse_velocity', - 'latitude', - 'longitude' + "fraud_score", + "country_code", + "region", + "city", + "zip_code", + "ISP", + "ASN", + "organization", + "is_crawler", + "timezone", + "mobile", + "host", + "proxy", + "vpn", + "tor", + "active_vpn", + "active_tor", + "recent_abuse", + "bot_status", + "connection_type", + "abuse_velocity", + "latitude", + "longitude", ] self.ip_data_items_friendly_names = { - 'fraud_score': 'IPQS: Fraud Score', - 'country_code': 'IPQS: Country Code', - 'region': 'IPQS: Region', - 'city': 'IPQS: City', - 'zip_code': 'IPQS: Zip Code', - 'ISP': 'IPQS: ISP', - 'ASN': 'IPQS: ASN', - 'organization': 'IPQS: Organization', - 'is_crawler': 'IPQS: Is Crawler', - 'timezone': 'IPQS: Timezone', - 'mobile': 'IPQS: Mobile', - 'host': 'IPQS: Host', - 'proxy': 'IPQS: Proxy', - 'vpn': 'IPQS: VPN', - 'tor': 'IPQS: TOR', - 'active_vpn': 'IPQS: Active VPN', - 'active_tor': 'IPQS: Active TOR', - 'recent_abuse': 'IPQS: Recent Abuse', - 'bot_status': 'IPQS: Bot Status', - 'connection_type': 'IPQS: Connection Type', - 'abuse_velocity': 'IPQS: Abuse Velocity', - 'latitude': 'IPQS: Latitude', - 'longitude': 'IPQS: Longitude' + "fraud_score": "IPQS: Fraud Score", + "country_code": "IPQS: Country Code", + "region": "IPQS: Region", + "city": "IPQS: City", + "zip_code": "IPQS: Zip Code", + "ISP": "IPQS: ISP", + "ASN": "IPQS: ASN", + "organization": "IPQS: Organization", + "is_crawler": "IPQS: Is Crawler", + "timezone": "IPQS: Timezone", + "mobile": "IPQS: Mobile", + "host": "IPQS: Host", + "proxy": "IPQS: Proxy", + "vpn": "IPQS: VPN", + "tor": "IPQS: TOR", + "active_vpn": "IPQS: Active VPN", + "active_tor": "IPQS: Active TOR", + "recent_abuse": "IPQS: Recent Abuse", + "bot_status": "IPQS: Bot Status", + "connection_type": "IPQS: Connection Type", + "abuse_velocity": "IPQS: Abuse Velocity", + "latitude": "IPQS: Latitude", + "longitude": "IPQS: Longitude", } self.url_data_items = [ - 'unsafe', - 'domain', - 'ip_address', - 'server', - 'domain_rank', - 'dns_valid', - 'parking', - 'spamming', - 'malware', - 'phishing', - 'suspicious', - 'adult', - 'risk_score', - 'category', - 'domain_age' + "unsafe", + "domain", + "ip_address", + "server", + "domain_rank", + "dns_valid", + "parking", + "spamming", + "malware", + "phishing", + "suspicious", + "adult", + "risk_score", + "category", + "domain_age", ] self.url_data_items_friendly_names = { - 'unsafe': 'IPQS: Unsafe', - 'domain': 'IPQS: Domain', - 'ip_address': 'IPQS: IP Address', - 'server': 'IPQS: Server', - 'domain_rank': 'IPQS: Domain Rank', - 'dns_valid': 'IPQS: DNS Valid', - 'parking': 'IPQS: Parking', - 'spamming': 'IPQS: Spamming', - 'malware': 'IPQS: Malware', - 'phishing': 'IPQS: Phishing', - 'suspicious': 'IPQS: Suspicious', - 'adult': 'IPQS: Adult', - 'risk_score': 'IPQS: Risk Score', - 'category': 'IPQS: Category', - 'domain_age': 'IPQS: Domain Age' + "unsafe": "IPQS: Unsafe", + "domain": "IPQS: Domain", + "ip_address": "IPQS: IP Address", + "server": "IPQS: Server", + "domain_rank": "IPQS: Domain Rank", + "dns_valid": "IPQS: DNS Valid", + "parking": "IPQS: Parking", + "spamming": "IPQS: Spamming", + "malware": "IPQS: Malware", + "phishing": "IPQS: Phishing", + "suspicious": "IPQS: Suspicious", + "adult": "IPQS: Adult", + "risk_score": "IPQS: Risk Score", + "category": "IPQS: Category", + "domain_age": "IPQS: Domain Age", } self.email_data_items = [ - 'valid', - 'disposable', - 'smtp_score', - 'overall_score', - 'first_name', - 'generic', - 'common', - 'dns_valid', - 'honeypot', - 'deliverability', - 'frequent_complainer', - 'spam_trap_score', - 'catch_all', - 'timed_out', - 'suspect', - 'recent_abuse', - 'fraud_score', - 'suggested_domain', - 'leaked', - 'sanitized_email', - 'domain_age', - 'first_seen' + "valid", + "disposable", + "smtp_score", + "overall_score", + "first_name", + "generic", + "common", + "dns_valid", + "honeypot", + "deliverability", + "frequent_complainer", + "spam_trap_score", + "catch_all", + "timed_out", + "suspect", + "recent_abuse", + "fraud_score", + "suggested_domain", + "leaked", + "sanitized_email", + "domain_age", + "first_seen", ] self.email_data_items_friendly_names = { - 'valid': 'IPQS: Valid', - 'disposable': 'IPQS: Disposable', - 'smtp_score': 'IPQS: SMTP Score', - 'overall_score': 'IPQS: Overall Score', - 'first_name': 'IPQS: First Name', - 'generic': 'IPQS: Generic', - 'common': 'IPQS: Common', - 'dns_valid': 'IPQS: DNS Valid', - 'honeypot': 'IPQS: Honeypot', - 'deliverability': 'IPQS: Deliverability', - 'frequent_complainer': 'IPQS: Frequent Complainer', - 'spam_trap_score': 'IPQS: Spam Trap Score', - 'catch_all': 'IPQS: Catch All', - 'timed_out': 'IPQS: Timed Out', - 'suspect': 'IPQS: Suspect', - 'recent_abuse': 'IPQS: Recent Abuse', - 'fraud_score': 'IPQS: Fraud Score', - 'suggested_domain': 'IPQS: Suggested Domain', - 'leaked': 'IPQS: Leaked', - 'sanitized_email': 'IPQS: Sanitized Email', - 'domain_age': 'IPQS: Domain Age', - 'first_seen': 'IPQS: First Seen' + "valid": "IPQS: Valid", + "disposable": "IPQS: Disposable", + "smtp_score": "IPQS: SMTP Score", + "overall_score": "IPQS: Overall Score", + "first_name": "IPQS: First Name", + "generic": "IPQS: Generic", + "common": "IPQS: Common", + "dns_valid": "IPQS: DNS Valid", + "honeypot": "IPQS: Honeypot", + "deliverability": "IPQS: Deliverability", + "frequent_complainer": "IPQS: Frequent Complainer", + "spam_trap_score": "IPQS: Spam Trap Score", + "catch_all": "IPQS: Catch All", + "timed_out": "IPQS: Timed Out", + "suspect": "IPQS: Suspect", + "recent_abuse": "IPQS: Recent Abuse", + "fraud_score": "IPQS: Fraud Score", + "suggested_domain": "IPQS: Suggested Domain", + "leaked": "IPQS: Leaked", + "sanitized_email": "IPQS: Sanitized Email", + "domain_age": "IPQS: Domain Age", + "first_seen": "IPQS: First Seen", } self.phone_data_items = [ - 'formatted', - 'local_format', - 'valid', - 'fraud_score', - 'recent_abuse', - 'VOIP', - 'prepaid', - 'risky', - 'active', - 'carrier', - 'line_type', - 'country', - 'city', - 'zip_code', - 'region', - 'dialing_code', - 'active_status', - 'leaked', - 'name', - 'timezone', - 'do_not_call', + "formatted", + "local_format", + "valid", + "fraud_score", + "recent_abuse", + "VOIP", + "prepaid", + "risky", + "active", + "carrier", + "line_type", + "country", + "city", + "zip_code", + "region", + "dialing_code", + "active_status", + "leaked", + "name", + "timezone", + "do_not_call", ] self.phone_data_items_friendly_names = { - 'formatted': 'IPQS: Formatted', - 'local_format': 'IPQS: Local Format', - 'valid': 'IPQS: Valid', - 'fraud_score': 'IPQS: Fraud Score', - 'recent_abuse': 'IPQS: Recent Abuse', - 'VOIP': 'IPQS: VOIP', - 'prepaid': 'IPQS: Prepaid', - 'risky': 'IPQS: Risky', - 'active': 'IPQS: Active', - 'carrier': 'IPQS: Carrier', - 'line_type': 'IPQS: Line Type', - 'country': 'IPQS: Country', - 'city': 'IPQS: City', - 'zip_code': 'IPQS: Zip Code', - 'region': 'IPQS: Region', - 'dialing_code': 'IPQS: Dialing Code', - 'active_status': 'IPQS: Active Status', - 'leaked': 'IPQS: Leaked', - 'name': 'IPQS: Name', - 'timezone': 'IPQS: Timezone', - 'do_not_call': 'IPQS: Do Not Call', + "formatted": "IPQS: Formatted", + "local_format": "IPQS: Local Format", + "valid": "IPQS: Valid", + "fraud_score": "IPQS: Fraud Score", + "recent_abuse": "IPQS: Recent Abuse", + "VOIP": "IPQS: VOIP", + "prepaid": "IPQS: Prepaid", + "risky": "IPQS: Risky", + "active": "IPQS: Active", + "carrier": "IPQS: Carrier", + "line_type": "IPQS: Line Type", + "country": "IPQS: Country", + "city": "IPQS: City", + "zip_code": "IPQS: Zip Code", + "region": "IPQS: Region", + "dialing_code": "IPQS: Dialing Code", + "active_status": "IPQS: Active Status", + "leaked": "IPQS: Leaked", + "name": "IPQS: Name", + "timezone": "IPQS: Timezone", + "do_not_call": "IPQS: Do Not Call", } self.timestamp_items_friendly_name = { - 'human': ' Human', - 'timestamp': ' Timestamp', - 'iso': ' ISO' + "human": " Human", + "timestamp": " Timestamp", + "iso": " ISO", } - self.timestamp_items = [ - 'human', - 'timestamp', - 'iso' - ] + self.timestamp_items = ["human", "timestamp", "iso"] def criticality_color(self, criticality) -> str: """method which maps the color to the criticality level""" @@ -357,7 +352,7 @@ def criticality_color(self, criticality) -> str: self.invalid: self.rf_red, self.disposable: self.rf_red, self.malware: self.rf_red, - self.phishing: self.rf_red + self.phishing: self.rf_red, } return mapper.get(criticality, self.rf_white) @@ -371,7 +366,7 @@ def add_tag(self, tag_name: str, hex_color: str = None) -> None: self.enriched_attribute.add_tag(tag) def ipqs_parser(self, query_response, enrich_type): - """ helper method to call the enrichment function according to the type""" + """helper method to call the enrichment function according to the type""" if enrich_type == IP_ENRICH: self.ip_reputation_data(query_response) elif enrich_type == URL_ENRICH: @@ -393,10 +388,8 @@ def ip_reputation_data(self, query_response): fraud_score = int(data_item_value) self.ip_address_risk_scoring(fraud_score) - self.ipqs_object.add_attribute( - "Enriched attribute", **self.enriched_attribute - ) - self.ipqs_object.add_reference(self.attribute['uuid'], 'related-to') + self.ipqs_object.add_attribute("Enriched attribute", **self.enriched_attribute) + self.ipqs_object.add_reference(self.attribute["uuid"], "related-to") self.misp_event.add_object(self.ipqs_object) def ip_address_risk_scoring(self, score): @@ -428,8 +421,10 @@ def url_reputation_data(self, query_response): data_item_value = "" if url_data_item == "domain_age": for timestamp_item in self.timestamp_items: - data_item = self.url_data_items_friendly_names[url_data_item] + \ - self.timestamp_items_friendly_name[timestamp_item] + data_item = ( + self.url_data_items_friendly_names[url_data_item] + + self.timestamp_items_friendly_name[timestamp_item] + ) data_item_value = str(query_response[url_data_item][timestamp_item]) self.ipqs_object.add_attribute(**parse_attribute(comment, data_item, data_item_value)) else: @@ -445,18 +440,16 @@ def url_reputation_data(self, query_response): risk_score = int(data_item_value) self.url_risk_scoring(risk_score, malware, phishing) - self.ipqs_object.add_attribute( - "Enriched attribute", **self.enriched_attribute - ) - self.ipqs_object.add_reference(self.attribute['uuid'], 'related-to') + self.ipqs_object.add_attribute("Enriched attribute", **self.enriched_attribute) + self.ipqs_object.add_reference(self.attribute["uuid"], "related-to") self.misp_event.add_object(self.ipqs_object) def url_risk_scoring(self, score, malware, phishing): """method to create calculate verdict for URL/Domain""" risk_criticality = "" - if malware == 'True': + if malware == "True": risk_criticality = self.malware - elif phishing == 'True': + elif phishing == "True": risk_criticality = self.phishing elif score >= 90: risk_criticality = self.high @@ -488,8 +481,10 @@ def email_reputation_data(self, query_response): self.ipqs_object.add_attribute(**parse_attribute(comment, data_item, data_item_value)) else: for timestamp_item in self.timestamp_items: - data_item = self.email_data_items_friendly_names[email_data_item] + \ - self.timestamp_items_friendly_name[timestamp_item] + data_item = ( + self.email_data_items_friendly_names[email_data_item] + + self.timestamp_items_friendly_name[timestamp_item] + ) data_item_value = str(query_response[email_data_item][timestamp_item]) self.ipqs_object.add_attribute(**parse_attribute(comment, data_item, data_item_value)) @@ -501,10 +496,8 @@ def email_reputation_data(self, query_response): fraud_score = int(data_item_value) self.email_address_risk_scoring(fraud_score, disposable, valid) - self.ipqs_object.add_attribute( - "Enriched attribute", **self.enriched_attribute - ) - self.ipqs_object.add_reference(self.attribute['uuid'], 'related-to') + self.ipqs_object.add_attribute("Enriched attribute", **self.enriched_attribute) + self.ipqs_object.add_reference(self.attribute["uuid"], "related-to") self.misp_event.add_object(self.ipqs_object) def email_address_risk_scoring(self, score, disposable, valid): @@ -545,12 +538,9 @@ def phone_reputation_data(self, query_response): if phone_data_item == "fraud_score": fraud_score = int(data_item_value) - self.phone_address_risk_scoring(fraud_score, valid, active) - self.ipqs_object.add_attribute( - "Enriched attribute", **self.enriched_attribute - ) - self.ipqs_object.add_reference(self.attribute['uuid'], 'related-to') + self.ipqs_object.add_attribute("Enriched attribute", **self.enriched_attribute) + self.ipqs_object.add_reference(self.attribute["uuid"], "related-to") self.misp_event.add_object(self.ipqs_object) def phone_address_risk_scoring(self, score, valid, active): @@ -575,32 +565,32 @@ def phone_address_risk_scoring(self, score, valid, active): def get_results(self): """returns the dictionary object to MISP Instance""" event = json.loads(self.misp_event.to_json()) - results = {key: event[key] for key in ('Attribute', 'Object')} - return {'results': results} + results = {key: event[key] for key in ("Attribute", "Object")} + return {"results": results} def handler(q=False): """The function which accepts a JSON document to expand the values and return a dictionary of the expanded - values. """ + values.""" if q is False: return False request = json.loads(q) # check if the apikey is provided - if not request.get('config') or not request['config'].get('apikey'): - misperrors['error'] = 'IPQualityScore apikey is missing' + if not request.get("config") or not request["config"].get("apikey"): + misperrors["error"] = "IPQualityScore apikey is missing" return misperrors - apikey = request['config'].get('apikey') + apikey = request["config"].get("apikey") # check attribute is added to the event - if not request.get('attribute') or not check_input_attribute(request['attribute']): - return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} + if not request.get("attribute") or not check_input_attribute(request["attribute"]): + return {"error": f"{standard_error_message}, which should contain at least a type, a value and an uuid."} - attribute = request['attribute'] - attribute_type = attribute['type'] - attribute_value = attribute['value'] + attribute = request["attribute"] + attribute_type = attribute["type"] + attribute_value = attribute["value"] # check if the attribute type is supported by IPQualityScore - if attribute_type not in mispattributes['input']: - return {'error': 'Unsupported attributes type for IPqualityScore Enrichment'} + if attribute_type not in mispattributes["input"]: + return {"error": "Unsupported attributes type for IPqualityScore Enrichment"} request_handler = RequestHandler(apikey) enrich_type = "" if attribute_type in ip_query_input_type: @@ -628,6 +618,6 @@ def introspection(): def version(): """The function that returns a dict with the version and the associated meta-data including potential - configurations required of the module. """ - moduleinfo['config'] = moduleconfig + configurations required of the module.""" + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/iprep.py b/misp_modules/modules/expansion/iprep.py index 8a5e959ca..68e3bf401 100755 --- a/misp_modules/modules/expansion/iprep.py +++ b/misp_modules/modules/expansion/iprep.py @@ -1,91 +1,118 @@ # -*- coding: utf-8 -*- import json -import requests +import requests -misperrors = {'error': 'Error'} -mispattributes = {'input': ['ip-src', 'ip-dst'], 'output': ['text']} +misperrors = {"error": "Error"} +mispattributes = {"input": ["ip-src", "ip-dst"], "output": ["text"]} moduleinfo = { - 'version': '1.0', - 'author': 'Keith Faber', - 'description': 'Module to query IPRep data for IP addresses.', - 'module-type': ['expansion'], - 'name': 'IPRep Lookup', - 'logo': '', - 'requirements': ['An access to the packetmail API (apikey)'], - 'features': 'This module takes an IP address attribute as input and queries the database from packetmail.net to get some information about the reputation of the IP.', - 'references': ['https://github.com/mahesh557/packetmail'], - 'input': 'An IP address MISP attribute.', - 'output': 'Text describing additional information about the input after a query on the IPRep API.', + "version": "1.0", + "author": "Keith Faber", + "description": "Module to query IPRep data for IP addresses.", + "module-type": ["expansion"], + "name": "IPRep Lookup", + "logo": "", + "requirements": ["An access to the packetmail API (apikey)"], + "features": ( + "This module takes an IP address attribute as input and queries the database from packetmail.net to get some" + " information about the reputation of the IP." + ), + "references": ["https://github.com/mahesh557/packetmail"], + "input": "An IP address MISP attribute.", + "output": "Text describing additional information about the input after a query on the IPRep API.", } -moduleconfig = ['apikey'] +moduleconfig = ["apikey"] def handler(q=False): if q is False: return False request = json.loads(q) - if request.get('ip-src'): - toquery = request['ip-src'] - elif request.get('ip-dst'): - toquery = request['ip-dst'] + if request.get("ip-src"): + toquery = request["ip-src"] + elif request.get("ip-dst"): + toquery = request["ip-dst"] else: - misperrors['error'] = "Unsupported attributes type" + misperrors["error"] = "Unsupported attributes type" return misperrors - if not request.get('config') and not request['config'].get('apikey'): - misperrors['error'] = 'IPRep api key is missing' + if not request.get("config") and not request["config"].get("apikey"): + misperrors["error"] = "IPRep api key is missing" return misperrors - err, rep = parse_iprep(toquery, request['config'].get('apikey')) + err, rep = parse_iprep(toquery, request["config"].get("apikey")) if len(err) > 0: - misperrors['error'] = ','.join(err) + misperrors["error"] = ",".join(err) return misperrors - return {'results': rep} + return {"results": rep} def parse_iprep(ip, api): - meta_fields = ['origin', 'Query_Time', 'created_on', 'IP_Lookup_History', 'IPs_in_collection', '_id', 'disclaimer', - 'MaxMind_Free_GeoIP', 'Unique_Lookups', 'query_result'] + meta_fields = [ + "origin", + "Query_Time", + "created_on", + "IP_Lookup_History", + "IPs_in_collection", + "_id", + "disclaimer", + "MaxMind_Free_GeoIP", + "Unique_Lookups", + "query_result", + ] rep = [] err = [] - full_text = '' - url = 'https://www.packetmail.net/iprep.php/%s' % ip + full_text = "" + url = "https://www.packetmail.net/iprep.php/%s" % ip try: - data = requests.get(url, params={'apikey': api}).json() + data = requests.get(url, params={"apikey": api}).json() except Exception: - return ['Error pulling data'], rep + return ["Error pulling data"], rep # print '%s' % data for name, val in data.items(): if name not in meta_fields: try: - context = val['context'] + context = val["context"] if type(context) is list: - if context[0].get('alert'): - context = ','.join([hit['alert']['signature'] for hit in context]) - elif context[0].get('signature'): - context = ','.join([hit['signature'] for hit in context]) - elif context[0].get('target_port') and context[0].get('protocol'): - context = ','.join( - ['Port Attacked: %s %s' % (hit['target_port'], hit['protocol']) for hit in context]) - elif context[0].get('phishing_kit') and context[0].get('url'): - context = ','.join(['%s (%s)' % (hit['phishing_kit'], hit['url']) for hit in context]) + if context[0].get("alert"): + context = ",".join([hit["alert"]["signature"] for hit in context]) + elif context[0].get("signature"): + context = ",".join([hit["signature"] for hit in context]) + elif context[0].get("target_port") and context[0].get("protocol"): + context = ",".join( + ["Port Attacked: %s %s" % (hit["target_port"], hit["protocol"]) for hit in context] + ) + elif context[0].get("phishing_kit") and context[0].get("url"): + context = ",".join(["%s (%s)" % (hit["phishing_kit"], hit["url"]) for hit in context]) else: - context = ';'.join(['%s: %s' % (k, v) for k, v in context[0].items()]) + context = ";".join(["%s: %s" % (k, v) for k, v in context[0].items()]) - if val.get('special_note'): - context += '; ' + val['special_note'] + if val.get("special_note"): + context += "; " + val["special_note"] misp_val = context - full_text += '\n%s' % context - misp_comment = 'IPRep Source %s: %s' % (name, val['last_seen']) - rep.append({'types': mispattributes['output'], 'categories': ['External analysis'], 'values': misp_val, 'comment': misp_comment}) + full_text += "\n%s" % context + misp_comment = "IPRep Source %s: %s" % (name, val["last_seen"]) + rep.append( + { + "types": mispattributes["output"], + "categories": ["External analysis"], + "values": misp_val, + "comment": misp_comment, + } + ) except Exception: - err.append('Error parsing source: %s' % name) - - rep.append({'types': ['freetext'], 'values': full_text, 'comment': 'Free text import of IPRep'}) + err.append("Error parsing source: %s" % name) + + rep.append( + { + "types": ["freetext"], + "values": full_text, + "comment": "Free text import of IPRep", + } + ) return err, rep @@ -94,5 +121,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/jinja_template_rendering.py b/misp_modules/modules/expansion/jinja_template_rendering.py index d65cb65bb..13d489aaf 100755 --- a/misp_modules/modules/expansion/jinja_template_rendering.py +++ b/misp_modules/modules/expansion/jinja_template_rendering.py @@ -1,48 +1,50 @@ #!/usr/bin/env python\ import json + from jinja2.sandbox import SandboxedEnvironment -misperrors = {'error': 'Error'} -mispattributes = {'input': ['text'], 'output': ['text']} +misperrors = {"error": "Error"} +mispattributes = {"input": ["text"], "output": ["text"]} moduleinfo = { - 'version': '0.1', - 'author': 'Sami Mokaddem', - 'description': 'Render the template with the data passed', - 'module-type': ['expansion'], - 'name': 'Ninja Template Rendering', - 'logo': '', - 'requirements': [], - 'features': '', - 'references': [], - 'input': '', - 'output': '', + "version": "0.1", + "author": "Sami Mokaddem", + "description": "Render the template with the data passed", + "module-type": ["expansion"], + "name": "Ninja Template Rendering", + "logo": "", + "requirements": [], + "features": "", + "references": [], + "input": "", + "output": "", } -default_template = '- Default template -' +default_template = "- Default template -" + def renderTemplate(data, template=default_template): env = SandboxedEnvironment() return env.from_string(template).render(data) + def handler(q=False): if q is False: return False request = json.loads(q) - if request.get('text'): - data = request['text'] + if request.get("text"): + data = request["text"] else: return False data = json.loads(data) - template = data.get('template', default_template) - templateData = data.get('data', {}) + template = data.get("template", default_template) + templateData = data.get("data", {}) try: rendered = renderTemplate(templateData, template) except TypeError: - rendered = '' + rendered = "" - r = {'results': [{'types': mispattributes['output'], - 'values':[rendered]}]} + r = {"results": [{"types": mispattributes["output"], "values": [rendered]}]} return r diff --git a/misp_modules/modules/expansion/joesandbox_query.py b/misp_modules/modules/expansion/joesandbox_query.py index f9c27a51a..d2e559cd0 100644 --- a/misp_modules/modules/expansion/joesandbox_query.py +++ b/misp_modules/modules/expansion/joesandbox_query.py @@ -1,91 +1,104 @@ # -*- coding: utf-8 -*- -import jbxapi import json -from . import check_input_attribute, checking_error, standard_error_message + +import jbxapi from joe_parser import JoeParser -misperrors = {'error': 'Error'} +from . import check_input_attribute, checking_error, standard_error_message + +misperrors = {"error": "Error"} -inputSource = ['link'] +inputSource = ["link"] moduleinfo = { - 'version': '0.2', - 'author': 'Christian Studer', - 'description': 'Query Joe Sandbox API with a submission url to get the json report and extract its data that is parsed and converted into MISP attributes and objects.', - 'module-type': ['expansion'], - 'name': 'Joe Sandbox Import', - 'logo': 'joesandbox.png', - 'requirements': ['jbxapi: Joe Sandbox API python3 library'], - 'features': "Module using the new format of modules able to return attributes and objects.\n\nThe module returns the same results as the import module [joe_import](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/joe_import.py) taking directly the json report as input.\n\nEven if the introspection will allow all kinds of links to call this module, obviously only the ones presenting a sample or url submission in the Joe Sandbox API will return results.\n\nTo make it work you will need to fill the 'apikey' configuration with your Joe Sandbox API key and provide a valid link as input.", - 'references': ['https://www.joesecurity.org', 'https://www.joesandbox.com/'], - 'input': 'Link of a Joe Sandbox sample or url submission.', - 'output': 'MISP attributes & objects parsed from the analysis report.', + "version": "0.2", + "author": "Christian Studer", + "description": ( + "Query Joe Sandbox API with a submission url to get the json report and extract its data that is parsed and" + " converted into MISP attributes and objects." + ), + "module-type": ["expansion"], + "name": "Joe Sandbox Import", + "logo": "joesandbox.png", + "requirements": ["jbxapi: Joe Sandbox API python3 library"], + "features": ( + "Module using the new format of modules able to return attributes and objects.\n\nThe module returns the same" + " results as the import module" + " [joe_import](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/joe_import.py)" + " taking directly the json report as input.\n\nEven if the introspection will allow all kinds of links to call" + " this module, obviously only the ones presenting a sample or url submission in the Joe Sandbox API will return" + " results.\n\nTo make it work you will need to fill the 'apikey' configuration with your Joe Sandbox API key" + " and provide a valid link as input." + ), + "references": ["https://www.joesecurity.org", "https://www.joesandbox.com/"], + "input": "Link of a Joe Sandbox sample or url submission.", + "output": "MISP attributes & objects parsed from the analysis report.", } -moduleconfig = ['apiurl', 'apikey', 'import_executable', 'import_mitre_attack'] +moduleconfig = ["apiurl", "apikey", "import_executable", "import_mitre_attack"] def handler(q=False): if q is False: return False request = json.loads(q) - apiurl = request['config'].get('apiurl') or 'https://jbxcloud.joesecurity.org/api' - apikey = request['config'].get('apikey') + apiurl = request["config"].get("apiurl") or "https://jbxcloud.joesecurity.org/api" + apikey = request["config"].get("apikey") parser_config = { - "import_executable": request["config"].get('import_executable', "false") == "true", - "mitre_attack": request["config"].get('import_mitre_attack', "false") == "true", + "import_executable": request["config"].get("import_executable", "false") == "true", + "mitre_attack": request["config"].get("import_mitre_attack", "false") == "true", } if not apikey: - return {'error': 'No API key provided'} + return {"error": "No API key provided"} - if not request.get('attribute') or not check_input_attribute(request['attribute'], requirements=('type', 'value')): - return {'error': f'{standard_error_message}, {checking_error} that is the link to the Joe Sandbox report.'} - if request['attribute']['type'] != 'link': - return {'error': 'Unsupported attribute type.'} - url = request['attribute']['value'] + if not request.get("attribute") or not check_input_attribute(request["attribute"], requirements=("type", "value")): + return {"error": f"{standard_error_message}, {checking_error} that is the link to the Joe Sandbox report."} + if request["attribute"]["type"] != "link": + return {"error": "Unsupported attribute type."} + url = request["attribute"]["value"] if "/submissions/" not in url: - return {'error': "The URL does not point to a Joe Sandbox analysis."} + return {"error": "The URL does not point to a Joe Sandbox analysis."} - submission_id = url.split('/')[-1] # The URL has the format https://example.net/submissions/12345 - joe = jbxapi.JoeSandbox(apiurl=apiurl, apikey=apikey, user_agent='MISP joesandbox_query') + submission_id = url.split("/")[-1] # The URL has the format https://example.net/submissions/12345 + joe = jbxapi.JoeSandbox(apiurl=apiurl, apikey=apikey, user_agent="MISP joesandbox_query") try: joe_info = joe.submission_info(submission_id) except jbxapi.ApiError as e: - return {'error': str(e)} + return {"error": str(e)} if joe_info["status"] != "finished": - return {'error': "The analysis has not finished yet."} + return {"error": "The analysis has not finished yet."} - if joe_info['most_relevant_analysis'] is None: - return {'error': "No analysis belongs to this submission."} + if joe_info["most_relevant_analysis"] is None: + return {"error": "No analysis belongs to this submission."} - analysis_webid = joe_info['most_relevant_analysis']['webid'] + analysis_webid = joe_info["most_relevant_analysis"]["webid"] joe_parser = JoeParser(parser_config) - joe_data = json.loads(joe.analysis_download(analysis_webid, 'jsonfixed')[1]) - joe_parser.parse_data(joe_data['analysis']) + joe_data = json.loads(joe.analysis_download(analysis_webid, "jsonfixed")[1]) + joe_parser.parse_data(joe_data["analysis"]) joe_parser.finalize_results() - return {'results': joe_parser.results} + return {"results": joe_parser.results} def introspection(): modulesetup = {} try: userConfig - modulesetup['userConfig'] = userConfig + modulesetup["userConfig"] = userConfig except NameError: pass try: inputSource - modulesetup['input'] = inputSource + modulesetup["input"] = inputSource except NameError: pass - modulesetup['format'] = 'misp_standard' + modulesetup["format"] = "misp_standard" return modulesetup def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/joesandbox_submit.py b/misp_modules/modules/expansion/joesandbox_submit.py index b124bb7c5..b80a702b5 100644 --- a/misp_modules/modules/expansion/joesandbox_submit.py +++ b/misp_modules/modules/expansion/joesandbox_submit.py @@ -1,37 +1,43 @@ -import jbxapi import base64 import io import json import logging +import re import sys import zipfile -import re - from urllib.parse import urljoin +import jbxapi log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) sh = logging.StreamHandler(sys.stdout) sh.setLevel(logging.DEBUG) -fmt = logging.Formatter( - "%(asctime)s - %(name)s - %(levelname)s - %(message)s" -) +fmt = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") sh.setFormatter(fmt) log.addHandler(sh) moduleinfo = { - 'version': '1.0', - 'author': 'Joe Security LLC', - 'description': 'A module to submit files or URLs to Joe Sandbox for an advanced analysis, and return the link of the submission.', - 'module-type': ['expansion', 'hover'], - 'name': 'Joe Sandbox Submit', - 'logo': 'joesandbox.png', - 'requirements': ['jbxapi: Joe Sandbox API python3 library'], - 'features': 'The module requires a Joe Sandbox API key to submit files or URL, and returns the link of the submitted analysis.\n\nIt is then possible, when the analysis is completed, to query the Joe Sandbox API to get the data related to the analysis, using the [joesandbox_query module](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/joesandbox_query.py) directly on this submission link.', - 'references': ['https://www.joesecurity.org', 'https://www.joesandbox.com/'], - 'input': 'Sample, url (or domain) to submit to Joe Sandbox for an advanced analysis.', - 'output': 'Link of the report generated in Joe Sandbox.', + "version": "1.0", + "author": "Joe Security LLC", + "description": ( + "A module to submit files or URLs to Joe Sandbox for an advanced analysis, and return the link of the" + " submission." + ), + "module-type": ["expansion", "hover"], + "name": "Joe Sandbox Submit", + "logo": "joesandbox.png", + "requirements": ["jbxapi: Joe Sandbox API python3 library"], + "features": ( + "The module requires a Joe Sandbox API key to submit files or URL, and returns the link of the submitted" + " analysis.\n\nIt is then possible, when the analysis is completed, to query the Joe Sandbox API to get the" + " data related to the analysis, using the [joesandbox_query" + " module](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/joesandbox_query.py)" + " directly on this submission link." + ), + "references": ["https://www.joesecurity.org", "https://www.joesandbox.com/"], + "input": "Sample, url (or domain) to submit to Joe Sandbox for an advanced analysis.", + "output": "Link of the report generated in Joe Sandbox.", } moduleconfig = [ "apiurl", @@ -74,7 +80,12 @@ def handler(q=False): if not apikey: return {"error": "No API key provided"} - joe = jbxapi.JoeSandbox(apiurl=apiurl, apikey=apikey, user_agent="MISP joesandbox_submit", accept_tac=accept_tac) + joe = jbxapi.JoeSandbox( + apiurl=apiurl, + apikey=apikey, + user_agent="MISP joesandbox_submit", + accept_tac=accept_tac, + ) try: is_url_submission = "url" in request or "domain" in request @@ -103,11 +114,13 @@ def handler(q=False): link_to_analysis = urljoin(apiurl, "../submissions/{}".format(result["submission_id"])) return { - "results": [{ - "types": "link", - "categories": "External analysis", - "values": link_to_analysis, - }] + "results": [ + { + "types": "link", + "categories": "External analysis", + "values": link_to_analysis, + } + ] } diff --git a/misp_modules/modules/expansion/lastline_query.py b/misp_modules/modules/expansion/lastline_query.py index 46310800d..f37339829 100644 --- a/misp_modules/modules/expansion/lastline_query.py +++ b/misp_modules/modules/expansion/lastline_query.py @@ -5,9 +5,10 @@ Module (type "expansion") to query a Lastline report from an analysis link. """ import json + import lastline_api -from . import check_input_attribute, checking_error, standard_error_message +from . import check_input_attribute, checking_error, standard_error_message misperrors = { "error": "Error", @@ -22,17 +23,25 @@ } moduleinfo = { - 'version': '0.1', - 'author': 'Stefano Ortolani', - 'description': 'Deprecation notice: this module will be deprecated by December 2021, please use vmware_nsx module.\n\nQuery Lastline with an analysis link and parse the report into MISP attributes and objects.', - 'module-type': ['expansion'], - 'name': 'Lastline Lookup', - 'logo': 'lastline.png', - 'requirements': [], - 'features': 'The module requires a Lastline Portal `username` and `password`.\nThe module uses the new format and it is able to return MISP attributes and objects.\nThe module returns the same results as the [lastline_import](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/lastline_import.py) import module.', - 'references': ['https://www.lastline.com'], - 'input': 'Link to a Lastline analysis.', - 'output': 'MISP attributes and objects parsed from the analysis report.', + "version": "0.1", + "author": "Stefano Ortolani", + "description": ( + "Deprecation notice: this module will be deprecated by December 2021, please use vmware_nsx module.\n\nQuery" + " Lastline with an analysis link and parse the report into MISP attributes and objects." + ), + "module-type": ["expansion"], + "name": "Lastline Lookup", + "logo": "lastline.png", + "requirements": [], + "features": ( + "The module requires a Lastline Portal `username` and `password`.\nThe module uses the new format and it is" + " able to return MISP attributes and objects.\nThe module returns the same results as the" + " [lastline_import](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/lastline_import.py)" + " import module." + ), + "references": ["https://www.lastline.com"], + "input": "Link to a Lastline analysis.", + "output": "MISP attributes and objects parsed from the analysis report.", } moduleconfig = [ @@ -61,9 +70,11 @@ def handler(q=False): try: config = request["config"] auth_data = lastline_api.LastlineAbstractClient.get_login_params_from_dict(config) - if not request.get('attribute') or not check_input_attribute(request['attribute'], requirements=('type', 'value')): - return {'error': f'{standard_error_message}, {checking_error} that is the link to a Lastline analysis.'} - analysis_link = request['attribute']['value'] + if not request.get("attribute") or not check_input_attribute( + request["attribute"], requirements=("type", "value") + ): + return {"error": f"{standard_error_message}, {checking_error} that is the link to a Lastline analysis."} + analysis_link = request["attribute"]["value"] # The API url changes based on the analysis link host name api_url = lastline_api.get_portal_url_from_task_link(analysis_link) except Exception as e: @@ -79,7 +90,11 @@ def handler(q=False): # Make the API calls try: - api_client = lastline_api.PortalClient(api_url, auth_data, verify_ssl=config.get('verify_ssl', True).lower() in ("true")) + api_client = lastline_api.PortalClient( + api_url, + auth_data, + verify_ssl=config.get("verify_ssl", True).lower() in ("true"), + ) response = api_client.get_progress(task_uuid) if response.get("completed") != 1: raise ValueError("Analysis is not finished yet.") @@ -101,9 +116,7 @@ def handler(q=False): return { "results": { - key: event_dictionary[key] - for key in ('Attribute', 'Object', 'Tag') - if (key in event and event[key]) + key: event_dictionary[key] for key in ("Attribute", "Object", "Tag") if (key in event and event[key]) } } @@ -125,11 +138,8 @@ def handler(q=False): { "config": a, "attribute": { - "value": ( - "https://user.lastline.com/portal#/analyst/task/" - "1fcbcb8f7fb400100772d6a7b62f501b/overview" - ) - } + "value": "https://user.lastline.com/portal#/analyst/task/1fcbcb8f7fb400100772d6a7b62f501b/overview" + }, } ) print(json.dumps(handler(j), indent=4, sort_keys=True)) @@ -138,11 +148,8 @@ def handler(q=False): { "config": a, "attribute": { - "value": ( - "https://user.lastline.com/portal#/analyst/task/" - "f3c0ae115d51001017ff8da768fa6049/overview" - ) - } + "value": "https://user.lastline.com/portal#/analyst/task/f3c0ae115d51001017ff8da768fa6049/overview" + }, } ) print(json.dumps(handler(j), indent=4, sort_keys=True)) diff --git a/misp_modules/modules/expansion/lastline_submit.py b/misp_modules/modules/expansion/lastline_submit.py index 52f15cfc2..050575000 100644 --- a/misp_modules/modules/expansion/lastline_submit.py +++ b/misp_modules/modules/expansion/lastline_submit.py @@ -11,7 +11,6 @@ import lastline_api - misperrors = { "error": "Error", } @@ -28,17 +27,25 @@ } moduleinfo = { - 'version': '0.1', - 'author': 'Stefano Ortolani', - 'description': 'Deprecation notice: this module will be deprecated by December 2021, please use vmware_nsx module.\n\nModule to submit a file or URL to Lastline.', - 'module-type': ['expansion', 'hover'], - 'name': 'Lastline Submit', - 'logo': 'lastline.png', - 'requirements': [], - 'features': 'The module requires a Lastline Analysis `api_token` and `key`.\nWhen the analysis is completed, it is possible to import the generated report by feeding the analysis link to the [lastline_query](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/lastline_query.py) module.', - 'references': ['https://www.lastline.com'], - 'input': 'File or URL to submit to Lastline.', - 'output': 'Link to the report generated by Lastline.', + "version": "0.1", + "author": "Stefano Ortolani", + "description": ( + "Deprecation notice: this module will be deprecated by December 2021, please use vmware_nsx module.\n\nModule" + " to submit a file or URL to Lastline." + ), + "module-type": ["expansion", "hover"], + "name": "Lastline Submit", + "logo": "lastline.png", + "requirements": [], + "features": ( + "The module requires a Lastline Analysis `api_token` and `key`.\nWhen the analysis is completed, it is possible" + " to import the generated report by feeding the analysis link to the" + " [lastline_query](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/lastline_query.py)" + " module." + ), + "references": ["https://www.lastline.com"], + "input": "File or URL to submit to Lastline.", + "output": "Link to the report generated by Lastline.", } moduleconfig = [ diff --git a/misp_modules/modules/expansion/macaddress_io.py b/misp_modules/modules/expansion/macaddress_io.py index 72e928e03..b6c1c8d9c 100644 --- a/misp_modules/modules/expansion/macaddress_io.py +++ b/misp_modules/modules/expansion/macaddress_io.py @@ -2,29 +2,37 @@ from maclookup import ApiClient, exceptions -misperrors = { - 'error': 'Error' -} +misperrors = {"error": "Error"} mispattributes = { - 'input': ['mac-address'], + "input": ["mac-address"], } moduleinfo = { - 'version': '1.0', - 'author': 'CodeLine OY - macaddress.io', - 'description': 'MISP hover module for macaddress.io', - 'module-type': ['hover'], - 'name': 'Macaddress.io Lookup', - 'logo': 'macaddress_io.png', - 'requirements': ['maclookup: macaddress.io python library', 'An access to the macaddress.io API (apikey)'], - 'features': 'This module takes a MAC address attribute as input and queries macaddress.io for additional information.\n\nThis information contains data about:\n- MAC address details\n- Vendor details\n- Block details', - 'references': ['https://macaddress.io/', 'https://github.com/CodeLineFi/maclookup-python'], - 'input': 'MAC address MISP attribute.', - 'output': 'Text containing information on the MAC address fetched from a query on macaddress.io.', + "version": "1.0", + "author": "CodeLine OY - macaddress.io", + "description": "MISP hover module for macaddress.io", + "module-type": ["hover"], + "name": "Macaddress.io Lookup", + "logo": "macaddress_io.png", + "requirements": [ + "maclookup: macaddress.io python library", + "An access to the macaddress.io API (apikey)", + ], + "features": ( + "This module takes a MAC address attribute as input and queries macaddress.io for additional" + " information.\n\nThis information contains data about:\n- MAC address details\n- Vendor details\n- Block" + " details" + ), + "references": [ + "https://macaddress.io/", + "https://github.com/CodeLineFi/maclookup-python", + ], + "input": "MAC address MISP attribute.", + "output": "Text containing information on the MAC address fetched from a query on macaddress.io.", } -moduleconfig = ['api_key'] +moduleconfig = ["api_key"] def handler(q=False): @@ -33,15 +41,15 @@ def handler(q=False): request = json.loads(q) - if request.get('mac-address'): - mac_address = request['mac-address'] + if request.get("mac-address"): + mac_address = request["mac-address"] else: return False - if request.get('config') and request['config'].get('api_key'): - api_key = request['config'].get('api_key') + if request.get("config") and request["config"].get("api_key"): + api_key = request["config"].get("api_key") else: - misperrors['error'] = 'Authorization required' + misperrors["error"] = "Authorization required" return misperrors api_client = ApiClient(api_key) @@ -50,72 +58,75 @@ def handler(q=False): response = api_client.get(mac_address) except exceptions.EmptyResponseException: - misperrors['error'] = 'Empty response' + misperrors["error"] = "Empty response" return misperrors except exceptions.UnparsableResponseException: - misperrors['error'] = 'Unparsable response' + misperrors["error"] = "Unparsable response" return misperrors except exceptions.ServerErrorException: - misperrors['error'] = 'Internal server error' + misperrors["error"] = "Internal server error" return misperrors except exceptions.UnknownOutputFormatException: - misperrors['error'] = 'Unknown output' + misperrors["error"] = "Unknown output" return misperrors except exceptions.AuthorizationRequiredException: - misperrors['error'] = 'Authorization required' + misperrors["error"] = "Authorization required" return misperrors except exceptions.AccessDeniedException: - misperrors['error'] = 'Access denied' + misperrors["error"] = "Access denied" return misperrors except exceptions.InvalidMacOrOuiException: - misperrors['error'] = 'Invalid MAC or OUI' + misperrors["error"] = "Invalid MAC or OUI" return misperrors except exceptions.NotEnoughCreditsException: - misperrors['error'] = 'Not enough credits' + misperrors["error"] = "Not enough credits" return misperrors except Exception: - misperrors['error'] = 'Unknown error' + misperrors["error"] = "Unknown error" return misperrors - date_created = \ - response.block_details.date_created.strftime('%d %B %Y') if response.block_details.date_created else None + date_created = ( + response.block_details.date_created.strftime("%d %B %Y") if response.block_details.date_created else None + ) - date_updated = \ - response.block_details.date_updated.strftime('%d %B %Y') if response.block_details.date_updated else None + date_updated = ( + response.block_details.date_updated.strftime("%d %B %Y") if response.block_details.date_updated else None + ) results = { - 'results': - [{'types': ['text'], 'values': + "results": [ { - # Mac address details - 'Valid MAC address': "True" if response.mac_address_details.is_valid else "False", - 'Transmission type': response.mac_address_details.transmission_type, - 'Administration type': response.mac_address_details.administration_type, - - # Vendor details - 'OUI': response.vendor_details.oui, - 'Vendor details are hidden': "True" if response.vendor_details.is_private else "False", - 'Company name': response.vendor_details.company_name, - 'Company\'s address': response.vendor_details.company_address, - 'County code': response.vendor_details.country_code, - - # Block details - 'Block found': "True" if response.block_details.block_found else "False", - 'The left border of the range': response.block_details.border_left, - 'The right border of the range': response.block_details.border_right, - 'The total number of MAC addresses in this range': response.block_details.block_size, - 'Assignment block size': response.block_details.assignment_block_size, - 'Date when the range was allocated': date_created, - 'Date when the range was last updated': date_updated - }}] + "types": ["text"], + "values": { + # Mac address details + "Valid MAC address": "True" if response.mac_address_details.is_valid else "False", + "Transmission type": response.mac_address_details.transmission_type, + "Administration type": response.mac_address_details.administration_type, + # Vendor details + "OUI": response.vendor_details.oui, + "Vendor details are hidden": "True" if response.vendor_details.is_private else "False", + "Company name": response.vendor_details.company_name, + "Company's address": response.vendor_details.company_address, + "County code": response.vendor_details.country_code, + # Block details + "Block found": "True" if response.block_details.block_found else "False", + "The left border of the range": response.block_details.border_left, + "The right border of the range": response.block_details.border_right, + "The total number of MAC addresses in this range": response.block_details.block_size, + "Assignment block size": response.block_details.assignment_block_size, + "Date when the range was allocated": date_created, + "Date when the range was last updated": date_updated, + }, + } + ] } return results @@ -126,5 +137,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/macvendors.py b/misp_modules/modules/expansion/macvendors.py index 3b21dd9e9..6ee0e8e9b 100644 --- a/misp_modules/modules/expansion/macvendors.py +++ b/misp_modules/modules/expansion/macvendors.py @@ -1,49 +1,57 @@ -import requests import json -misperrors = {'error': 'Error'} -mispattributes = {'input': ['mac-address'], 'output': ['text']} +import requests + +misperrors = {"error": "Error"} +mispattributes = {"input": ["mac-address"], "output": ["text"]} moduleinfo = { - 'version': '0.1', - 'author': 'Aurélien Schwab', - 'description': 'Module to access Macvendors API.', - 'module-type': ['hover'], - 'name': 'Macvendors Lookup', - 'logo': 'macvendors.png', - 'requirements': [], - 'features': 'The module takes a MAC address as input and queries macvendors.com for some information about it. The API returns the name of the vendor related to the address.', - 'references': ['https://macvendors.com/', 'https://macvendors.com/api'], - 'input': 'A MAC address.', - 'output': 'Additional information about the MAC address.', + "version": "0.1", + "author": "Aurélien Schwab", + "description": "Module to access Macvendors API.", + "module-type": ["hover"], + "name": "Macvendors Lookup", + "logo": "macvendors.png", + "requirements": [], + "features": ( + "The module takes a MAC address as input and queries macvendors.com for some information about it. The API" + " returns the name of the vendor related to the address." + ), + "references": ["https://macvendors.com/", "https://macvendors.com/api"], + "input": "A MAC address.", + "output": "Additional information about the MAC address.", } -moduleconfig = ['user-agent'] +moduleconfig = ["user-agent"] -macvendors_api_url = 'https://api.macvendors.com/' -default_user_agent = 'MISP-Module' +macvendors_api_url = "https://api.macvendors.com/" +default_user_agent = "MISP-Module" def handler(q=False): if q is False: return False request = json.loads(q) - for input_type in mispattributes['input']: + for input_type in mispattributes["input"]: if input_type in request: mac = request[input_type] break else: - misperrors['error'] = "Unsupported attributes type" + misperrors["error"] = "Unsupported attributes type" return misperrors - user_agent = request['config']['user-agent'] if request.get('config') and request['config'].get('user-agent') else default_user_agent - r = requests.get(macvendors_api_url + mac, headers={'user-agent': user_agent}) # Real request + user_agent = ( + request["config"]["user-agent"] + if request.get("config") and request["config"].get("user-agent") + else default_user_agent + ) + r = requests.get(macvendors_api_url + mac, headers={"user-agent": user_agent}) # Real request if r.status_code == 200: # OK (record found) response = r.text if response: - return {'results': [{'types': mispattributes['output'], 'values': response}]} + return {"results": [{"types": mispattributes["output"], "values": response}]} elif r.status_code == 404: # Not found (not an error) - return {'results': [{'types': mispattributes['output'], 'values': 'Not found'}]} + return {"results": [{"types": mispattributes["output"], "values": "Not found"}]} else: # Real error - misperrors['error'] = 'MacVendors API not accessible (HTTP ' + str(r.status_code) + ')' - return misperrors['error'] + misperrors["error"] = "MacVendors API not accessible (HTTP " + str(r.status_code) + ")" + return misperrors["error"] def introspection(): @@ -51,5 +59,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/malshare_upload.py b/misp_modules/modules/expansion/malshare_upload.py index b810e2095..f41d584a7 100644 --- a/misp_modules/modules/expansion/malshare_upload.py +++ b/misp_modules/modules/expansion/malshare_upload.py @@ -1,25 +1,25 @@ -import json -import sys import base64 +import hashlib import io +import json +import re import zipfile + import requests -import hashlib -import re -misperrors = {'error': 'Error'} -mispattributes = {'input': ['attachment', 'malware-sample'], 'output': ['link']} +misperrors = {"error": "Error"} +mispattributes = {"input": ["attachment", "malware-sample"], "output": ["link"]} moduleinfo = { - 'version': '1', - 'author': 'Karen Yousefi', - 'description': 'Module to push malware samples to MalShare', - 'module-type': ['expansion'], - 'name': 'MalShare Upload', - 'requirements': ['requests library'], - 'logo': '', + "version": "1", + "author": "Karen Yousefi", + "description": "Module to push malware samples to MalShare", + "module-type": ["expansion"], + "name": "MalShare Upload", + "requirements": ["requests library"], + "logo": "", } -moduleconfig = ['malshare_apikey'] +moduleconfig = ["malshare_apikey"] def handler(q=False): @@ -29,7 +29,7 @@ def handler(q=False): try: data = request.get("data") - if 'malware-sample' in request: + if "malware-sample" in request: sample_filename = request.get("malware-sample").split("|", 1)[0] data = base64.b64decode(data) fl = io.BytesIO(data) @@ -37,14 +37,14 @@ def handler(q=False): sample_hashname = zf.namelist()[0] data = zf.read(sample_hashname, b"infected") zf.close() - elif 'attachment' in request: + elif "attachment" in request: sample_filename = request.get("attachment") data = base64.b64decode(data) else: - misperrors['error'] = "No malware sample or attachment supplied" + misperrors["error"] = "No malware sample or attachment supplied" return misperrors except Exception: - misperrors['error'] = "Unable to process submitted sample data" + misperrors["error"] = "Unable to process submitted sample data" return misperrors if request["config"].get("malshare_apikey") is None: @@ -55,7 +55,7 @@ def handler(q=False): try: url = "https://malshare.com/api.php" - params = {'api_key': malshare_apikey, 'action': 'upload'} + params = {"api_key": malshare_apikey, "action": "upload"} files = {"upload": (sample_filename, data)} response = requests.post(url, params=params, files=files) response.raise_for_status() @@ -67,31 +67,27 @@ def handler(q=False): if response_text.startswith("Success"): # If upload was successful or file already exists - malshare_link = ( - f"https://malshare.com/sample.php?action=detail&hash={sha256}" - ) + malshare_link = f"https://malshare.com/sample.php?action=detail&hash={sha256}" elif "sample already exists" in response_text: # If file already exists, extract SHA256 from response - match = re.search(r'([a-fA-F0-9]{64})', response_text) + match = re.search(r"([a-fA-F0-9]{64})", response_text) if match: sha256 = match.group(1) - malshare_link = ( - f"https://malshare.com/sample.php?action=detail&hash={sha256}" - ) + malshare_link = f"https://malshare.com/sample.php?action=detail&hash={sha256}" else: # If there's any other error raise Exception(f"Upload failed: {response_text}") except Exception as e: - misperrors['error'] = f"Unable to send sample to MalShare: {str(e)}" + misperrors["error"] = f"Unable to send sample to MalShare: {str(e)}" return misperrors r = { - 'results': [ + "results": [ { - 'types': 'link', - 'values': malshare_link, - 'comment': 'Link to MalShare analysis', + "types": "link", + "values": malshare_link, + "comment": "Link to MalShare analysis", } ] } @@ -103,5 +99,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/malwarebazaar.py b/misp_modules/modules/expansion/malwarebazaar.py index 5ad90477c..330235389 100644 --- a/misp_modules/modules/expansion/malwarebazaar.py +++ b/misp_modules/modules/expansion/malwarebazaar.py @@ -1,61 +1,75 @@ import json + import requests -from . import check_input_attribute, checking_error, standard_error_message from pymisp import MISPEvent, MISPObject -mispattributes = {'input': ['md5', 'sha1', 'sha256'], - 'format': 'misp_standard'} +from . import check_input_attribute, checking_error, standard_error_message + +mispattributes = {"input": ["md5", "sha1", "sha256"], "format": "misp_standard"} moduleinfo = { - 'version': '0.1', - 'author': 'Christian Studer', - 'description': 'Query Malware Bazaar to get additional information about the input hash.', - 'module-type': ['expansion', 'hover'], - 'name': 'Malware Bazaar Lookup', - 'logo': '', - 'requirements': [], - 'features': "The module takes a hash attribute as input and queries MALWAREbazaar's API to fetch additional data about it. The result, if the payload is known on the databases, is at least one file object describing the file the input hash is related to.\n\nThe module is using the new format of modules able to return object since the result is one or multiple MISP object(s).", - 'references': ['https://bazaar.abuse.ch/'], - 'input': 'A hash attribute (md5, sha1 or sha256).', - 'output': 'File object(s) related to the input attribute found on MALWAREbazaar databases.', + "version": "0.1", + "author": "Christian Studer", + "description": "Query Malware Bazaar to get additional information about the input hash.", + "module-type": ["expansion", "hover"], + "name": "Malware Bazaar Lookup", + "logo": "", + "requirements": [], + "features": ( + "The module takes a hash attribute as input and queries MALWAREbazaar's API to fetch additional data about it." + " The result, if the payload is known on the databases, is at least one file object describing the file the" + " input hash is related to.\n\nThe module is using the new format of modules able to return object since the" + " result is one or multiple MISP object(s)." + ), + "references": ["https://bazaar.abuse.ch/"], + "input": "A hash attribute (md5, sha1 or sha256).", + "output": "File object(s) related to the input attribute found on MALWAREbazaar databases.", } moduleconfig = [] def parse_response(response): - mapping = {'file_name': {'type': 'filename', 'object_relation': 'filename'}, - 'file_size': {'type': 'size-in-bytes', 'object_relation': 'size-in-bytes'}, - 'file_type_mime': {'type': 'mime-type', 'object_relation': 'mimetype'}, - 'md5_hash': {'type': 'md5', 'object_relation': 'md5'}, - 'sha1_hash': {'type': 'sha1', 'object_relation': 'sha1'}, - 'sha256_hash': {'type': 'sha256', 'object_relation': 'sha256'}, - 'ssdeep': {'type': 'ssdeep', 'object_relation': 'ssdeep'}} + mapping = { + "file_name": {"type": "filename", "object_relation": "filename"}, + "file_size": {"type": "size-in-bytes", "object_relation": "size-in-bytes"}, + "file_type_mime": {"type": "mime-type", "object_relation": "mimetype"}, + "md5_hash": {"type": "md5", "object_relation": "md5"}, + "sha1_hash": {"type": "sha1", "object_relation": "sha1"}, + "sha256_hash": {"type": "sha256", "object_relation": "sha256"}, + "ssdeep": {"type": "ssdeep", "object_relation": "ssdeep"}, + } misp_event = MISPEvent() for data in response: - misp_object = MISPObject('file') + misp_object = MISPObject("file") for feature, attribute in mapping.items(): if feature in data: - misp_attribute = {'value': data[feature]} + misp_attribute = {"value": data[feature]} misp_attribute.update(attribute) misp_object.add_attribute(**misp_attribute) misp_event.add_object(**misp_object) - return {'results': {'Object': [json.loads(misp_object.to_json()) for misp_object in misp_event.objects]}} + return {"results": {"Object": [json.loads(misp_object.to_json()) for misp_object in misp_event.objects]}} def handler(q=False): if q is False: return False request = json.loads(q) - if not request.get('attribute') or not check_input_attribute(request['attribute'], requirements=('type', 'value')): - return {'error': f'{standard_error_message}, {checking_error} that is the hash to submit to Malware Bazaar.'} - attribute = request['attribute'] - if attribute['type'] not in mispattributes['input']: - return {'error': 'Unsupported attribute type.'} - url = 'https://mb-api.abuse.ch/api/v1/' - response = requests.post(url, data={'query': 'get_info', 'hash': attribute['value']}).json() - query_status = response['query_status'] - if query_status == 'ok': - return parse_response(response['data']) - return {'error': 'Hash not found on MALWAREbazzar' if query_status == 'hash_not_found' else f'Problem encountered during the query: {query_status}'} + if not request.get("attribute") or not check_input_attribute(request["attribute"], requirements=("type", "value")): + return {"error": f"{standard_error_message}, {checking_error} that is the hash to submit to Malware Bazaar."} + attribute = request["attribute"] + if attribute["type"] not in mispattributes["input"]: + return {"error": "Unsupported attribute type."} + url = "https://mb-api.abuse.ch/api/v1/" + response = requests.post(url, data={"query": "get_info", "hash": attribute["value"]}).json() + query_status = response["query_status"] + if query_status == "ok": + return parse_response(response["data"]) + return { + "error": ( + "Hash not found on MALWAREbazzar" + if query_status == "hash_not_found" + else f"Problem encountered during the query: {query_status}" + ) + } def introspection(): @@ -63,5 +77,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/mcafee_insights_enrich.py b/misp_modules/modules/expansion/mcafee_insights_enrich.py index dca67b8be..2a0a4f00c 100644 --- a/misp_modules/modules/expansion/mcafee_insights_enrich.py +++ b/misp_modules/modules/expansion/mcafee_insights_enrich.py @@ -3,43 +3,41 @@ import json import logging -import requests import sys -from . import check_input_attribute, standard_error_message +import requests from pymisp import MISPAttribute, MISPEvent, MISPObject -misperrors = {'error': 'Error'} -mispattributes = {'input': ["md5", "sha1", "sha256"], - 'format': 'misp_standard'} +misperrors = {"error": "Error"} +mispattributes = {"input": ["md5", "sha1", "sha256"], "format": "misp_standard"} # possible module-types: 'expansion', 'hover' or both moduleinfo = { - 'version': '1', - 'author': 'Martin Ohl', - 'description': 'Lookup McAfee MVISION Insights Details', - 'module-type': ['hover'], - 'name': 'McAfee MVISION Insights Lookup', - 'logo': '', - 'requirements': [], - 'features': '', - 'references': [], - 'input': '', - 'output': '', + "version": "1", + "author": "Martin Ohl", + "description": "Lookup McAfee MVISION Insights Details", + "module-type": ["hover"], + "name": "McAfee MVISION Insights Lookup", + "logo": "", + "requirements": [], + "features": "", + "references": [], + "input": "", + "output": "", } # config fields that your code expects from the site admin -moduleconfig = ['api_key', 'client_id', 'client_secret'] +moduleconfig = ["api_key", "client_id", "client_secret"] -class MVAPI(): +class MVAPI: def __init__(self, attribute, api_key, client_id, client_secret): self.misp_event = MISPEvent() self.attribute = MISPAttribute() self.attribute.from_dict(**attribute) self.misp_event.add_attribute(**self.attribute) - self.base_url = 'https://api.mvision.mcafee.com' + self.base_url = "https://api.mvision.mcafee.com" self.session = requests.Session() self.api_key = api_key @@ -49,8 +47,8 @@ def __init__(self, attribute, api_key, client_id, client_secret): self.auth(auth) def logging(self): - self.logger = logging.getLogger('logs') - self.logger.setLevel('INFO') + self.logger = logging.getLogger("logs") + self.logger.setLevel("INFO") handler = logging.StreamHandler() formatter = logging.Formatter("%(asctime)s;%(levelname)s;%(message)s") handler.setFormatter(formatter) @@ -60,163 +58,168 @@ def auth(self, auth): iam_url = "https://iam.mcafee-cloud.com/iam/v1.1/token" headers = { - 'x-api-key': self.api_key, - 'Content-Type': 'application/vnd.api+json' + "x-api-key": self.api_key, + "Content-Type": "application/vnd.api+json", } payload = { "grant_type": "client_credentials", - "scope": "ins.user ins.suser ins.ms.r" + "scope": "ins.user ins.suser ins.ms.r", } res = self.session.post(iam_url, headers=headers, auth=auth, data=payload) if res.status_code != 200: - self.logger.error('Could not authenticate to get the IAM token: {0} - {1}'.format(res.status_code, res.text)) + self.logger.error( + "Could not authenticate to get the IAM token: {0} - {1}".format(res.status_code, res.text) + ) sys.exit() else: - self.logger.info('Successful authenticated.') - access_token = res.json()['access_token'] - headers['Authorization'] = 'Bearer ' + access_token + self.logger.info("Successful authenticated.") + access_token = res.json()["access_token"] + headers["Authorization"] = "Bearer " + access_token self.session.headers = headers def search_ioc(self): filters = { - 'filter[type][eq]': self.attribute.type, - 'filter[value]': self.attribute.value, - 'fields': 'id, type, value, coverage, uid, is_coat, is_sdb_dirty, category, comment, campaigns, threat, prevalence' + "filter[type][eq]": self.attribute.type, + "filter[value]": self.attribute.value, + "fields": ( + "id, type, value, coverage, uid, is_coat, is_sdb_dirty, category, comment, campaigns, threat," + " prevalence" + ), } - res = self.session.get(self.base_url + '/insights/v2/iocs', params=filters) + res = self.session.get(self.base_url + "/insights/v2/iocs", params=filters) if res.ok: - if len(res.json()['data']) == 0: - self.logger.info('No Hash details in MVISION Insights found.') + if len(res.json()["data"]) == 0: + self.logger.info("No Hash details in MVISION Insights found.") else: - self.logger.info('Successfully retrieved MVISION Insights details.') + self.logger.info("Successfully retrieved MVISION Insights details.") self.logger.debug(res.text) return res.json() else: - self.logger.error('Error in search_ioc. HTTP {0} - {1}'.format(str(res.status_code), res.text)) + self.logger.error("Error in search_ioc. HTTP {0} - {1}".format(str(res.status_code), res.text)) sys.exit() def prep_result(self, ioc): - res = ioc['data'][0] + res = ioc["data"][0] results = [] # Parse out Attribute Category category_attr = { - 'type': 'text', - 'object_relation': 'text', - 'value': 'Attribute Category: {0}'.format(res['attributes']['category']) + "type": "text", + "object_relation": "text", + "value": "Attribute Category: {0}".format(res["attributes"]["category"]), } results.append(category_attr) # Parse out Attribute Comment comment_attr = { - 'type': 'text', - 'object_relation': 'text', - 'value': 'Attribute Comment: {0}'.format(res['attributes']['comment']) + "type": "text", + "object_relation": "text", + "value": "Attribute Comment: {0}".format(res["attributes"]["comment"]), } results.append(comment_attr) # Parse out Attribute Dat Coverage cover_attr = { - 'type': 'text', - 'object_relation': 'text', - 'value': 'Dat Version Coverage: {0}'.format(res['attributes']['coverage']['dat_version']['min']) + "type": "text", + "object_relation": "text", + "value": "Dat Version Coverage: {0}".format(res["attributes"]["coverage"]["dat_version"]["min"]), } results.append(cover_attr) # Parse out if Dirty cover_attr = { - 'type': 'text', - 'object_relation': 'text', - 'value': 'Is Dirty: {0}'.format(res['attributes']['is-sdb-dirty']) + "type": "text", + "object_relation": "text", + "value": "Is Dirty: {0}".format(res["attributes"]["is-sdb-dirty"]), } results.append(cover_attr) # Parse our targeted countries countries_dict = [] - countries = res['attributes']['prevalence']['countries'] + countries = res["attributes"]["prevalence"]["countries"] for country in countries: - countries_dict.append(country['iso_code']) + countries_dict.append(country["iso_code"]) country_attr = { - 'type': 'text', - 'object_relation': 'text', - 'value': 'Targeted Countries: {0}'.format(countries_dict) + "type": "text", + "object_relation": "text", + "value": "Targeted Countries: {0}".format(countries_dict), } results.append(country_attr) # Parse out targeted sectors sectors_dict = [] - sectors = res['attributes']['prevalence']['sectors'] + sectors = res["attributes"]["prevalence"]["sectors"] for sector in sectors: - sectors_dict.append(sector['sector']) + sectors_dict.append(sector["sector"]) sector_attr = { - 'type': 'text', - 'object_relation': 'text', - 'value': 'Targeted Sectors: {0}'.format(sectors_dict) + "type": "text", + "object_relation": "text", + "value": "Targeted Sectors: {0}".format(sectors_dict), } results.append(sector_attr) # Parse out Threat Classification threat_class_attr = { - 'type': 'text', - 'object_relation': 'text', - 'value': 'Threat Classification: {0}'.format(res['attributes']['threat']['classification']) + "type": "text", + "object_relation": "text", + "value": "Threat Classification: {0}".format(res["attributes"]["threat"]["classification"]), } results.append(threat_class_attr) # Parse out Threat Name threat_name_attr = { - 'type': 'text', - 'object_relation': 'text', - 'value': 'Threat Name: {0}'.format(res['attributes']['threat']['name']) + "type": "text", + "object_relation": "text", + "value": "Threat Name: {0}".format(res["attributes"]["threat"]["name"]), } results.append(threat_name_attr) # Parse out Threat Severity threat_sev_attr = { - 'type': 'text', - 'object_relation': 'text', - 'value': 'Threat Severity: {0}'.format(res['attributes']['threat']['severity']) + "type": "text", + "object_relation": "text", + "value": "Threat Severity: {0}".format(res["attributes"]["threat"]["severity"]), } results.append(threat_sev_attr) # Parse out Attribute ID attr_id = { - 'type': 'text', - 'object_relation': 'text', - 'value': 'Attribute ID: {0}'.format(res['id']) + "type": "text", + "object_relation": "text", + "value": "Attribute ID: {0}".format(res["id"]), } results.append(attr_id) # Parse out Campaign Relationships - campaigns = ioc['included'] + campaigns = ioc["included"] for campaign in campaigns: campaign_attr = { - 'type': 'campaign-name', - 'object_relation': 'campaign-name', - 'value': campaign['attributes']['name'] + "type": "campaign-name", + "object_relation": "campaign-name", + "value": campaign["attributes"]["name"], } results.append(campaign_attr) - mv_insights_obj = MISPObject(name='MVISION Insights Details') + mv_insights_obj = MISPObject(name="MVISION Insights Details") for mvi_res in results: mv_insights_obj.add_attribute(**mvi_res) - mv_insights_obj.add_reference(self.attribute.uuid, 'mvision-insights-details') + mv_insights_obj.add_reference(self.attribute.uuid, "mvision-insights-details") self.misp_event.add_object(mv_insights_obj) event = json.loads(self.misp_event.to_json()) - results_mvi = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} + results_mvi = {key: event[key] for key in ("Attribute", "Object") if (key in event and event[key])} - return {'results': results_mvi} + return {"results": results_mvi} def handler(q=False): @@ -224,16 +227,21 @@ def handler(q=False): return False request = json.loads(q) - if not request.get('config') or not request['config'].get('api_key') or not request['config'].get('client_id') or not request['config'].get('client_secret'): - misperrors['error'] = "Please provide MVISION API Key, Client ID and Client Secret." + if ( + not request.get("config") + or not request["config"].get("api_key") + or not request["config"].get("client_id") + or not request["config"].get("client_secret") + ): + misperrors["error"] = "Please provide MVISION API Key, Client ID and Client Secret." return misperrors - if request['attribute']['type'] not in mispattributes['input']: - return {'error': 'Unsupported attribute type. Please use {0}'.format(mispattributes['input'])} + if request["attribute"]["type"] not in mispattributes["input"]: + return {"error": "Unsupported attribute type. Please use {0}".format(mispattributes["input"])} - api_key = request['config']['api_key'] - client_id = request['config']['client_id'] - client_secret = request['config']['client_secret'] - attribute = request['attribute'] + api_key = request["config"]["api_key"] + client_id = request["config"]["client_id"] + client_secret = request["config"]["client_secret"] + attribute = request["attribute"] mvi = MVAPI(attribute, api_key, client_id, client_secret) res = mvi.search_ioc() @@ -245,5 +253,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/mmdb_lookup.py b/misp_modules/modules/expansion/mmdb_lookup.py index 93de1ce1c..87864357f 100644 --- a/misp_modules/modules/expansion/mmdb_lookup.py +++ b/misp_modules/modules/expansion/mmdb_lookup.py @@ -1,26 +1,43 @@ import json + import requests -from . import check_input_attribute, standard_error_message from pymisp import MISPEvent, MISPObject -misperrors = {'error': 'Error'} -mispattributes = {'input': ['ip-src', 'ip-src|port', 'ip-dst', 'ip-dst|port'], 'format': 'misp_standard'} -moduleinfo = {'version': '1', - 'author': 'Jeroen Pinoy', - 'description': "A hover and expansion module to enrich an ip with geolocation and ASN information from an mmdb server instance, such as CIRCL's ip.circl.lu.", - 'module-type': ['expansion', 'hover'], - 'name': 'GeoIP Enrichment', - 'logo': 'circl.png', - 'requirements': [], - 'features': 'The module takes an IP address related attribute as input.\n It queries the public CIRCL.lu mmdb-server instance, available at ip.circl.lu, by default. The module can be configured with a custom mmdb server url if required.\n It is also possible to filter results on 1 db_source by configuring db_source_filter.', - 'references': ['https://data.public.lu/fr/datasets/geo-open-ip-address-geolocation-per-country-in-mmdb-format/', 'https://github.com/adulau/mmdb-server'], - 'input': 'An IP address attribute (for example ip-src or ip-src|port).', - 'output': 'Geolocation and asn objects.'} +from . import check_input_attribute, standard_error_message + +misperrors = {"error": "Error"} +mispattributes = { + "input": ["ip-src", "ip-src|port", "ip-dst", "ip-dst|port"], + "format": "misp_standard", +} +moduleinfo = { + "version": "1", + "author": "Jeroen Pinoy", + "description": ( + "A hover and expansion module to enrich an ip with geolocation and ASN information from an mmdb server" + " instance, such as CIRCL's ip.circl.lu." + ), + "module-type": ["expansion", "hover"], + "name": "GeoIP Enrichment", + "logo": "circl.png", + "requirements": [], + "features": ( + "The module takes an IP address related attribute as input.\n It queries the public CIRCL.lu mmdb-server" + " instance, available at ip.circl.lu, by default. The module can be configured with a custom mmdb server url if" + " required.\n It is also possible to filter results on 1 db_source by configuring db_source_filter." + ), + "references": [ + "https://data.public.lu/fr/datasets/geo-open-ip-address-geolocation-per-country-in-mmdb-format/", + "https://github.com/adulau/mmdb-server", + ], + "input": "An IP address attribute (for example ip-src or ip-src|port).", + "output": "Geolocation and asn objects.", +} moduleconfig = ["custom_API", "db_source_filter", "max_country_info_qt"] -mmdblookup_url = 'https://ip.circl.lu/' +mmdblookup_url = "https://ip.circl.lu/" -class MmdbLookupParser(): +class MmdbLookupParser: def __init__(self, attribute, mmdblookupresult, api_url, max_country_info_qt=0): self.attribute = attribute self.mmdblookupresult = mmdblookupresult @@ -31,102 +48,137 @@ def __init__(self, attribute, mmdblookupresult, api_url, max_country_info_qt=0): def get_result(self): event = json.loads(self.misp_event.to_json()) - results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} - return {'results': results} + results = {key: event[key] for key in ("Attribute", "Object") if (key in event and event[key])} + return {"results": results} def parse_mmdblookup_information(self): # There is a chance some db's have a hit while others don't so we have to check if entry is empty each time country_info_qt = 0 for result_entry in self.mmdblookupresult: - if result_entry['country_info']: - if (self.max_country_info_qt == 0) or (self.max_country_info_qt > 0 and country_info_qt < self.max_country_info_qt): - mmdblookup_object = MISPObject('geolocation') - mmdblookup_object.add_attribute('country', - **{'type': 'text', 'value': result_entry['country_info']['Country']}) - mmdblookup_object.add_attribute('countrycode', - **{'type': 'text', 'value': result_entry['country']['iso_code']}) - mmdblookup_object.add_attribute('latitude', - **{'type': 'float', - 'value': result_entry['country_info']['Latitude (average)']}) - mmdblookup_object.add_attribute('longitude', - **{'type': 'float', - 'value': result_entry['country_info']['Longitude (average)']}) - mmdblookup_object.add_attribute('text', - **{'type': 'text', - 'value': 'db_source: {}. build_db: {}. Latitude and longitude are country average.'.format( - result_entry['meta']['db_source'], - result_entry['meta']['build_db'])}) - mmdblookup_object.add_reference(self.attribute['uuid'], 'related-to') + if result_entry["country_info"]: + if (self.max_country_info_qt == 0) or ( + self.max_country_info_qt > 0 and country_info_qt < self.max_country_info_qt + ): + mmdblookup_object = MISPObject("geolocation") + mmdblookup_object.add_attribute( + "country", + **{ + "type": "text", + "value": result_entry["country_info"]["Country"], + }, + ) + mmdblookup_object.add_attribute( + "countrycode", + **{ + "type": "text", + "value": result_entry["country"]["iso_code"], + }, + ) + mmdblookup_object.add_attribute( + "latitude", + **{ + "type": "float", + "value": result_entry["country_info"]["Latitude (average)"], + }, + ) + mmdblookup_object.add_attribute( + "longitude", + **{ + "type": "float", + "value": result_entry["country_info"]["Longitude (average)"], + }, + ) + mmdblookup_object.add_attribute( + "text", + **{ + "type": "text", + "value": "db_source: {}. build_db: {}. Latitude and longitude are country average.".format( + result_entry["meta"]["db_source"], + result_entry["meta"]["build_db"], + ), + }, + ) + mmdblookup_object.add_reference(self.attribute["uuid"], "related-to") self.misp_event.add_object(mmdblookup_object) country_info_qt += 1 - if 'AutonomousSystemNumber' in result_entry['country']: - mmdblookup_object_asn = MISPObject('asn') - mmdblookup_object_asn.add_attribute('asn', - **{'type': 'text', - 'value': result_entry['country'][ - 'AutonomousSystemNumber']}) - mmdblookup_object_asn.add_attribute('description', - **{'type': 'text', - 'value': 'ASNOrganization: {}. db_source: {}. build_db: {}.'.format( - result_entry['country'][ - 'AutonomousSystemOrganization'], - result_entry['meta']['db_source'], - result_entry['meta']['build_db'])}) - mmdblookup_object_asn.add_reference(self.attribute['uuid'], 'related-to') + if "AutonomousSystemNumber" in result_entry["country"]: + mmdblookup_object_asn = MISPObject("asn") + mmdblookup_object_asn.add_attribute( + "asn", + **{ + "type": "text", + "value": result_entry["country"]["AutonomousSystemNumber"], + }, + ) + mmdblookup_object_asn.add_attribute( + "description", + **{ + "type": "text", + "value": "ASNOrganization: {}. db_source: {}. build_db: {}.".format( + result_entry["country"]["AutonomousSystemOrganization"], + result_entry["meta"]["db_source"], + result_entry["meta"]["build_db"], + ), + }, + ) + mmdblookup_object_asn.add_reference(self.attribute["uuid"], "related-to") self.misp_event.add_object(mmdblookup_object_asn) def check_url(url): - return "{}/".format(url) if not url.endswith('/') else url + return "{}/".format(url) if not url.endswith("/") else url def handler(q=False): if q is False: return False request = json.loads(q) - if not request.get('attribute') or not check_input_attribute(request['attribute']): - return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} - attribute = request['attribute'] - if attribute.get('type') == 'ip-src': - toquery = attribute['value'] - elif attribute.get('type') == 'ip-src|port': - toquery = attribute['value'].split('|')[0] - elif attribute.get('type') == 'ip-dst': - toquery = attribute['value'] - elif attribute.get('type') == 'ip-dst|port': - toquery = attribute['value'].split('|')[0] + if not request.get("attribute") or not check_input_attribute(request["attribute"]): + return {"error": f"{standard_error_message}, which should contain at least a type, a value and an uuid."} + attribute = request["attribute"] + if attribute.get("type") == "ip-src": + toquery = attribute["value"] + elif attribute.get("type") == "ip-src|port": + toquery = attribute["value"].split("|")[0] + elif attribute.get("type") == "ip-dst": + toquery = attribute["value"] + elif attribute.get("type") == "ip-dst|port": + toquery = attribute["value"].split("|")[0] else: - misperrors['error'] = 'There is no attribute of type ip-src or ip-dst provided as input' + misperrors["error"] = "There is no attribute of type ip-src or ip-dst provided as input" return misperrors - max_country_info_qt = request['config'].get('max_country_info_qt', 0) + max_country_info_qt = request["config"].get("max_country_info_qt", 0) if max_country_info_qt is None: max_country_info_qt = 0 - api_url = check_url(request['config']['custom_API']) if 'config' in request and request['config'].get( - 'custom_API') else mmdblookup_url + api_url = ( + check_url(request["config"]["custom_API"]) + if "config" in request and request["config"].get("custom_API") + else mmdblookup_url + ) r = requests.get("{}/geolookup/{}".format(api_url, toquery)) if r.status_code == 200: mmdblookupresult = r.json() if not mmdblookupresult or len(mmdblookupresult) == 0: - misperrors['error'] = 'Empty result returned by server' + misperrors["error"] = "Empty result returned by server" return misperrors - if 'config' in request and request['config'].get('db_source_filter'): - db_source_filter = request['config'].get('db_source_filter') - mmdblookupresult = [entry for entry in mmdblookupresult if entry['meta']['db_source'] == db_source_filter] + if "config" in request and request["config"].get("db_source_filter"): + db_source_filter = request["config"].get("db_source_filter") + mmdblookupresult = [entry for entry in mmdblookupresult if entry["meta"]["db_source"] == db_source_filter] if not mmdblookupresult or len(mmdblookupresult) == 0: - misperrors['error'] = 'There was no result with the selected db_source' + misperrors["error"] = "There was no result with the selected db_source" return misperrors # Server might return one or multiple entries which could all be empty, we check if there is at least one # non-empty result below empty_result = True for lookup_result_entry in mmdblookupresult: - if lookup_result_entry['country_info']: + if lookup_result_entry["country_info"]: empty_result = False break if empty_result: - misperrors['error'] = 'Empty result returned by server' + misperrors["error"] = "Empty result returned by server" return misperrors else: - misperrors['error'] = 'API not accessible - http status code {} was returned'.format(r.status_code) + misperrors["error"] = "API not accessible - http status code {} was returned".format(r.status_code) return misperrors parser = MmdbLookupParser(attribute, mmdblookupresult, api_url, max_country_info_qt) parser.parse_mmdblookup_information() @@ -139,5 +191,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/mwdb.py b/misp_modules/modules/expansion/mwdb.py index a6fdc1e64..550bb42c7 100644 --- a/misp_modules/modules/expansion/mwdb.py +++ b/misp_modules/modules/expansion/mwdb.py @@ -1,31 +1,48 @@ -import json -import sys import base64 -#from distutils.util import strtobool - import io +import json +import sys import zipfile -from pymisp import PyMISP from mwdblib import MWDB +from pymisp import PyMISP + +# from distutils.util import strtobool + -misperrors = {'error': 'Error'} -mispattributes = {'input': ['attachment', 'malware-sample'], 'output': ['link']} +misperrors = {"error": "Error"} +mispattributes = {"input": ["attachment", "malware-sample"], "output": ["link"]} moduleinfo = { - 'version': '1', - 'author': 'Koen Van Impe', - 'description': 'Module to push malware samples to a MWDB instance', - 'module-type': ['expansion'], - 'name': 'MWDB Submit', - 'logo': '', - 'requirements': ['* mwdblib installed (pip install mwdblib) ; * (optional) keys.py file to add tags of events/attributes to MWDB * (optional) MWDB attribute created for the link back to MISP (defined in mwdb_misp_attribute)'], - 'features': 'An expansion module to push malware samples to a MWDB (https://github.com/CERT-Polska/mwdb-core) instance. This module does not push samples to a sandbox. This can be achieved via Karton (connected to the MWDB). Does: * Upload of attachment or malware sample to MWDB * Tags of events and/or attributes are added to MWDB. * Comment of the MISP attribute is added to MWDB. * A link back to the MISP event is added to MWDB via the MWDB attribute. * A link to the MWDB attribute is added as an enrichted attribute to the MISP event.', - 'references': [], - 'input': 'Attachment or malware sample', - 'output': 'Link attribute that points to the sample at the MWDB instane', + "version": "1", + "author": "Koen Van Impe", + "description": "Module to push malware samples to a MWDB instance", + "module-type": ["expansion"], + "name": "MWDB Submit", + "logo": "", + "requirements": [ + "* mwdblib installed (pip install mwdblib) ; * (optional) keys.py file to add tags of events/attributes to MWDB" + " * (optional) MWDB attribute created for the link back to MISP (defined in mwdb_misp_attribute)" + ], + "features": ( + "An expansion module to push malware samples to a MWDB (https://github.com/CERT-Polska/mwdb-core) instance." + " This module does not push samples to a sandbox. This can be achieved via Karton (connected to the MWDB)." + " Does: * Upload of attachment or malware sample to MWDB * Tags of events and/or attributes are added to MWDB." + " * Comment of the MISP attribute is added to MWDB. * A link back to the MISP event is added to MWDB via the" + " MWDB attribute. * A link to the MWDB attribute is added as an enrichted attribute to the MISP event." + ), + "references": [], + "input": "Attachment or malware sample", + "output": "Link attribute that points to the sample at the MWDB instane", } -moduleconfig = ['mwdb_apikey', 'mwdb_url', 'mwdb_misp_attribute', 'mwdb_public', 'include_tags_event', 'include_tags_attribute'] +moduleconfig = [ + "mwdb_apikey", + "mwdb_url", + "mwdb_misp_attribute", + "mwdb_public", + "include_tags_event", + "include_tags_attribute", +] pymisp_keys_file = "/var/www/MISP/PyMISP/" mwdb_public_default = True @@ -55,7 +72,7 @@ def handler(q=False): try: data = request.get("data") - if 'malware-sample' in request: + if "malware-sample" in request: # malicious samples are encrypted with zip (password infected) and then base64 encoded sample_filename = request.get("malware-sample").split("|", 1)[0] data = base64.b64decode(data) @@ -64,16 +81,16 @@ def handler(q=False): sample_hashname = zf.namelist()[0] data = zf.read(sample_hashname, b"infected") zf.close() - elif 'attachment' in request: + elif "attachment" in request: # All attachments get base64 encoded sample_filename = request.get("attachment") data = base64.b64decode(data) else: - misperrors['error'] = "No malware sample or attachment supplied" + misperrors["error"] = "No malware sample or attachment supplied" return misperrors except Exception: - misperrors['error'] = "Unable to process submited sample data" + misperrors["error"] = "Unable to process submited sample data" return misperrors if (request["config"].get("mwdb_apikey") is None) or (request["config"].get("mwdb_url") is None): @@ -94,7 +111,8 @@ def handler(q=False): try: if include_tags_event: sys.path.append(pymisp_keys_file) - from keys import misp_url, misp_key, misp_verifycert + from keys import misp_key, misp_url, misp_verifycert + misp = PyMISP(misp_url, misp_key, misp_verifycert, False) misp_event = misp.get_event(misp_event_id) if "Event" in misp_event: @@ -106,7 +124,8 @@ def handler(q=False): mwdb_tags.append(tag["name"]) if include_tags_attribute: sys.path.append(pymisp_keys_file) - from keys import misp_url, misp_key, misp_verifycert + from keys import misp_key, misp_url, misp_verifycert + misp = PyMISP(misp_url, misp_key, misp_verifycert, False) misp_attribute = misp.get_attribute(misp_attribute_uuid) if "Attribute" in misp_attribute: @@ -117,11 +136,14 @@ def handler(q=False): mwdb_tags.append(tag["name"]) misp_attribute_comment = misp_attribute["Attribute"]["comment"] except Exception: - misperrors['error'] = "Unable to read PyMISP (keys.py) configuration file" + misperrors["error"] = "Unable to read PyMISP (keys.py) configuration file" return misperrors try: - mwdb = MWDB(api_key=request["config"].get("mwdb_apikey"), api_url=request["config"].get("mwdb_url")) + mwdb = MWDB( + api_key=request["config"].get("mwdb_apikey"), + api_url=request["config"].get("mwdb_url"), + ) if mwdb_misp_attribute and len(mwdb_misp_attribute) > 0: metakeys = {mwdb_misp_attribute: misp_event_id} else: @@ -136,10 +158,10 @@ def handler(q=False): file_object.add_comment("Fetched from event {} - {}".format(misp_event_id, misp_info)) mwdb_link = request["config"].get("mwdb_url").replace("/api", "/file/") + "{}".format(file_object.md5) except Exception: - misperrors['error'] = "Unable to send sample to MWDB instance" + misperrors["error"] = "Unable to send sample to MWDB instance" return misperrors - r = {'results': [{'types': 'link', 'values': mwdb_link, 'comment': 'Link to MWDB sample'}]} + r = {"results": [{"types": "link", "values": mwdb_link, "comment": "Link to MWDB sample"}]} return r @@ -148,5 +170,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/ocr_enrich.py b/misp_modules/modules/expansion/ocr_enrich.py index 0fbaea4c2..4f804d6a8 100644 --- a/misp_modules/modules/expansion/ocr_enrich.py +++ b/misp_modules/modules/expansion/ocr_enrich.py @@ -1,32 +1,35 @@ -import json import binascii +import json + import cv2 import np import pytesseract -misperrors = {'error': 'Error'} -mispattributes = {'input': ['attachment'], - 'output': ['freetext']} +misperrors = {"error": "Error"} +mispattributes = {"input": ["attachment"], "output": ["freetext"]} moduleinfo = { - 'version': '0.2', - 'author': 'Sascha Rommelfangen', - 'description': 'Module to process some optical character recognition on pictures.', - 'module-type': ['expansion'], - 'name': 'OCR Enrich', - 'logo': '', - 'requirements': ['cv2: The OpenCV python library.'], - 'features': 'The module takes an attachment attributes as input and process some optical character recognition on it. The text found is then passed to the Freetext importer to extract potential IoCs.', - 'references': [], - 'input': 'A picture attachment.', - 'output': 'Text and freetext fetched from the input picture.', + "version": "0.2", + "author": "Sascha Rommelfangen", + "description": "Module to process some optical character recognition on pictures.", + "module-type": ["expansion"], + "name": "OCR Enrich", + "logo": "", + "requirements": ["cv2: The OpenCV python library."], + "features": ( + "The module takes an attachment attributes as input and process some optical character recognition on it. The" + " text found is then passed to the Freetext importer to extract potential IoCs." + ), + "references": [], + "input": "A picture attachment.", + "output": "Text and freetext fetched from the input picture.", } moduleconfig = [] def filter_decoded(decoded): - for line in decoded.split('\n'): - decoded_line = line.strip('\t\x0b\x0c\r ') + for line in decoded.split("\n"): + decoded_line = line.strip("\t\x0b\x0c\r ") if decoded_line: yield decoded_line @@ -35,13 +38,13 @@ def handler(q=False): if q is False: return False q = json.loads(q) - filename = q['attachment'] + filename = q["attachment"] try: - img_array = np.frombuffer(binascii.a2b_base64(q['data']), np.uint8) + img_array = np.frombuffer(binascii.a2b_base64(q["data"]), np.uint8) except Exception as e: print(e) err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?" - misperrors['error'] = err + misperrors["error"] = err print(err) return misperrors @@ -50,18 +53,18 @@ def handler(q=False): try: decoded = pytesseract.image_to_string(cv2.cvtColor(image, cv2.COLOR_BGR2RGB)) return { - 'results': [ + "results": [ { - 'types': ['freetext'], - 'values': list(filter_decoded(decoded)), - 'comment': f"OCR from file {filename}" + "types": ["freetext"], + "values": list(filter_decoded(decoded)), + "comment": f"OCR from file {filename}", } ] } except Exception as e: print(e) err = "Couldn't analyze file type. Only images are supported right now." - misperrors['error'] = err + misperrors["error"] = err return misperrors @@ -70,5 +73,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/ods_enrich.py b/misp_modules/modules/expansion/ods_enrich.py index 6d25f321b..5774ec9e0 100644 --- a/misp_modules/modules/expansion/ods_enrich.py +++ b/misp_modules/modules/expansion/ods_enrich.py @@ -1,26 +1,32 @@ -import json import binascii -import np -import ezodf -import pandas_ods_reader import io +import json import logging -misperrors = {'error': 'Error'} -mispattributes = {'input': ['attachment'], - 'output': ['freetext', 'text']} +import ezodf +import np +import pandas_ods_reader + +misperrors = {"error": "Error"} +mispattributes = {"input": ["attachment"], "output": ["freetext", "text"]} moduleinfo = { - 'version': '0.1', - 'author': 'Sascha Rommelfangen', - 'description': 'Module to extract freetext from a .ods document.', - 'module-type': ['expansion'], - 'name': 'ODS Enrich', - 'logo': 'ods.png', - 'requirements': ['ezodf: Python package to create/manipulate OpenDocumentFormat files.', 'pandas_ods_reader: Python library to read in ODS files.'], - 'features': 'The module reads the text contained in a .ods document. The result is passed to the freetext import parser so IoCs can be extracted out of it.', - 'references': [], - 'input': 'Attachment attribute containing a .ods document.', - 'output': 'Text and freetext parsed from the document.', + "version": "0.1", + "author": "Sascha Rommelfangen", + "description": "Module to extract freetext from a .ods document.", + "module-type": ["expansion"], + "name": "ODS Enrich", + "logo": "ods.png", + "requirements": [ + "ezodf: Python package to create/manipulate OpenDocumentFormat files.", + "pandas_ods_reader: Python library to read in ODS files.", + ], + "features": ( + "The module reads the text contained in a .ods document. The result is passed to the freetext import parser so" + " IoCs can be extracted out of it." + ), + "references": [], + "input": "Attachment attribute containing a .ods document.", + "output": "Text and freetext parsed from the document.", } moduleconfig = [] @@ -30,13 +36,13 @@ def handler(q=False): if q is False: return False q = json.loads(q) - filename = q['attachment'] + filename = q["attachment"] try: - ods_array = np.frombuffer(binascii.a2b_base64(q['data']), np.uint8) + ods_array = np.frombuffer(binascii.a2b_base64(q["data"]), np.uint8) except Exception as e: print(e) err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?" - misperrors['error'] = err + misperrors["error"] = err print(err) return misperrors @@ -48,17 +54,35 @@ def handler(q=False): for i in range(0, num_sheets): rows = pandas_ods_reader.parsers.ods.get_rows(doc, i) try: - ods = pandas_ods_reader.algo.parse_data(pandas_ods_reader.parsers.ods, rows, headers=False, columns=[], skiprows=0) + ods = pandas_ods_reader.algo.parse_data( + pandas_ods_reader.parsers.ods, + rows, + headers=False, + columns=[], + skiprows=0, + ) ods = pandas_ods_reader.utils.sanitize_df(ods) except TypeError: ods = pandas_ods_reader.algo.read_data(pandas_ods_reader.parsers.ods, ods_file, i, headers=False) ods_content = ods_content + "\n" + ods.to_string(max_rows=None) - return {'results': [{'types': ['freetext'], 'values': ods_content, 'comment': ".ods-to-text from file " + filename}, - {'types': ['text'], 'values': ods_content, 'comment': ".ods-to-text from file " + filename}]} + return { + "results": [ + { + "types": ["freetext"], + "values": ods_content, + "comment": ".ods-to-text from file " + filename, + }, + { + "types": ["text"], + "values": ods_content, + "comment": ".ods-to-text from file " + filename, + }, + ] + } except Exception as e: logging.exception(e) err = "Couldn't analyze file as .ods. Error was: " + str(e) - misperrors['error'] = err + misperrors["error"] = err return misperrors @@ -67,5 +91,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/odt_enrich.py b/misp_modules/modules/expansion/odt_enrich.py index 04d120f2e..e724baa16 100644 --- a/misp_modules/modules/expansion/odt_enrich.py +++ b/misp_modules/modules/expansion/odt_enrich.py @@ -1,24 +1,27 @@ -import json import binascii +import io +import json + import np from ODTReader.odtreader import odtToText -import io -misperrors = {'error': 'Error'} -mispattributes = {'input': ['attachment'], - 'output': ['freetext', 'text']} +misperrors = {"error": "Error"} +mispattributes = {"input": ["attachment"], "output": ["freetext", "text"]} moduleinfo = { - 'version': '0.1', - 'author': 'Sascha Rommelfangen', - 'description': 'Module to extract freetext from a .odt document.', - 'module-type': ['expansion'], - 'name': 'ODT Enrich', - 'logo': 'odt.png', - 'requirements': ['ODT reader python library.'], - 'features': 'The module reads the text contained in a .odt document. The result is passed to the freetext import parser so IoCs can be extracted out of it.', - 'references': [], - 'input': 'Attachment attribute containing a .odt document.', - 'output': 'Text and freetext parsed from the document.', + "version": "0.1", + "author": "Sascha Rommelfangen", + "description": "Module to extract freetext from a .odt document.", + "module-type": ["expansion"], + "name": "ODT Enrich", + "logo": "odt.png", + "requirements": ["ODT reader python library."], + "features": ( + "The module reads the text contained in a .odt document. The result is passed to the freetext import parser so" + " IoCs can be extracted out of it." + ), + "references": [], + "input": "Attachment attribute containing a .odt document.", + "output": "Text and freetext parsed from the document.", } moduleconfig = [] @@ -28,13 +31,13 @@ def handler(q=False): if q is False: return False q = json.loads(q) - filename = q['attachment'] + filename = q["attachment"] try: - odt_array = np.frombuffer(binascii.a2b_base64(q['data']), np.uint8) + odt_array = np.frombuffer(binascii.a2b_base64(q["data"]), np.uint8) except Exception as e: print(e) err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?" - misperrors['error'] = err + misperrors["error"] = err print(err) return misperrors @@ -43,12 +46,24 @@ def handler(q=False): try: odt_content = odtToText(odt_file) print(odt_content) - return {'results': [{'types': ['freetext'], 'values': odt_content, 'comment': ".odt-to-text from file " + filename}, - {'types': ['text'], 'values': odt_content, 'comment': ".odt-to-text from file " + filename}]} + return { + "results": [ + { + "types": ["freetext"], + "values": odt_content, + "comment": ".odt-to-text from file " + filename, + }, + { + "types": ["text"], + "values": odt_content, + "comment": ".odt-to-text from file " + filename, + }, + ] + } except Exception as e: print(e) err = "Couldn't analyze file as .odt. Error was: " + str(e) - misperrors['error'] = err + misperrors["error"] = err return misperrors @@ -57,5 +72,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/onion_lookup.py b/misp_modules/modules/expansion/onion_lookup.py index 17dcc89a2..447ed5e6c 100644 --- a/misp_modules/modules/expansion/onion_lookup.py +++ b/misp_modules/modules/expansion/onion_lookup.py @@ -1,34 +1,35 @@ import json import requests -from pymisp import MISPEvent, MISPObject, MISPAttribute +from pymisp import MISPAttribute, MISPEvent, MISPObject from . import check_input_attribute, standard_error_message -misperrors = {'error': 'Error'} +misperrors = {"error": "Error"} mispattributes = { - 'input': [ - 'onion-address', + "input": [ + "onion-address", # 'domain', # 'ip-dst', # 'url', # Any other Attribute type... ], - 'format': 'misp_standard', + "format": "misp_standard", } moduleinfo = { - 'version': '1', - 'author': 'Sami Mokaddem', - 'name': 'Onion Lookup', - 'author': 'MISP', - 'description': 'MISP module using the MISP standard. Uses the onion-lookup service to get information about an onion.', - 'module-type': [ # possible module-types: 'expansion', 'hover' or both - 'expansion', - 'hover', + "version": "1", + "author": "Sami Mokaddem", + "name": "Onion Lookup", + "description": ( + "MISP module using the MISP standard. Uses the onion-lookup service to get information about an onion." + ), + "module-type": [ # possible module-types: 'expansion', 'hover' or both + "expansion", + "hover", ], - 'references': ['https://onion.ail-project.org/'], - 'logo': 'onion.png' + "references": ["https://onion.ail-project.org/"], + "logo": "onion.png", } # config fields that your code expects from the site admin @@ -36,12 +37,12 @@ def getDetails(onion_address): - url = f'https://onion.ail-project.org/api/lookup/{onion_address}' + url = f"https://onion.ail-project.org/api/lookup/{onion_address}" response = requests.get(url) return response.json() -''' +""" { "tags": [ "infoleak:automatic-detection=\"base64\"", @@ -49,36 +50,36 @@ def getDetails(onion_address): "infoleak:automatic-detection=\"onion\"" ], } -''' +""" def createObject(onion_details): - misp_object = MISPObject('tor-hiddenservice') - misp_object.comment = 'custom-comment2' - onion_address = misp_object.add_attribute('address', onion_details['id']) - misp_object.add_attribute('first-seen', onion_details['first_seen']) - misp_object.add_attribute('last-seen', onion_details['last_seen']) - for lang in onion_details['languages']: - misp_object.add_attribute('language', lang) - for title in onion_details['titles']: - misp_object.add_attribute('title', title) - for tag in onion_details['tags']: + misp_object = MISPObject("tor-hiddenservice") + misp_object.comment = "custom-comment2" + onion_address = misp_object.add_attribute("address", onion_details["id"]) + misp_object.add_attribute("first-seen", onion_details["first_seen"]) + misp_object.add_attribute("last-seen", onion_details["last_seen"]) + for lang in onion_details["languages"]: + misp_object.add_attribute("language", lang) + for title in onion_details["titles"]: + misp_object.add_attribute("title", title) + for tag in onion_details["tags"]: onion_address.add_tag(tag) return misp_object def enrichOnion(misp_event, attribute): - onion_address = attribute['value'] + onion_address = attribute["value"] onion_details = getDetails(onion_address) misp_object = createObject(onion_details) misp_event.add_object(misp_object) original_attribute = MISPAttribute() original_attribute.from_dict(**attribute) - original_attribute.comment = 'custom comment' - for tag in onion_details['tags']: + original_attribute.comment = "custom comment" + for tag in onion_details["tags"]: original_attribute.add_tag(tag) misp_event.attributes.append(original_attribute) - misp_object.add_reference(attribute['uuid'], 'expanded-from') + misp_object.add_reference(attribute["uuid"], "expanded-from") return misp_event @@ -88,15 +89,13 @@ def handler(q=False): request = json.loads(q) # Input sanity check - if not request.get('attribute') or not check_input_attribute(request['attribute']): - return { - 'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.' - } - attribute = request['attribute'] + if not request.get("attribute") or not check_input_attribute(request["attribute"]): + return {"error": f"{standard_error_message}, which should contain at least a type, a value and an uuid."} + attribute = request["attribute"] # Make sure the Attribute's type is one of the expected type - if attribute['type'] not in mispattributes['input']: - return {'error': 'Unsupported attribute type.'} + if attribute["type"] not in mispattributes["input"]: + return {"error": "Unsupported attribute type."} # Use PyMISP to create compatible MISP Format misp_event = MISPEvent() @@ -105,10 +104,10 @@ def handler(q=False): # Convert to the format understood by MISP results = {} event = misp_event.to_dict() - for key in ('Attribute', 'Object', 'EventReport'): + for key in ("Attribute", "Object", "EventReport"): if key in event: results[key] = event[key] - return {'results': results} + return {"results": results} def introspection(): @@ -116,5 +115,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/onyphe.py b/misp_modules/modules/expansion/onyphe.py index 29213b465..c2f66c437 100644 --- a/misp_modules/modules/expansion/onyphe.py +++ b/misp_modules/modules/expansion/onyphe.py @@ -4,33 +4,35 @@ from pymisp import MISPEvent, MISPObject -try: - from onyphe import Onyphe -except ImportError: - print("pyonyphe module not installed.") +from misp_modules.lib.onyphe import Onyphe -misperrors = {'error': 'Error'} +misperrors = {"error": "Error"} -mispattributes = {'input': ['ip-src', 'ip-dst', 'hostname', 'domain'], - 'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'url'], - 'format': 'misp_standard'} +mispattributes = { + "input": ["ip-src", "ip-dst", "hostname", "domain"], + "output": ["hostname", "domain", "ip-src", "ip-dst", "url"], + "format": "misp_standard", +} # possible module-types: 'expansion', 'hover' or both moduleinfo = { - 'version': '2', - 'author': 'Sebastien Larinier @sebdraven', - 'description': 'Module to process a query on Onyphe.', - 'module-type': ['expansion', 'hover'], - 'name': 'Onyphe Lookup', - 'logo': 'onyphe.jpg', - 'requirements': ['onyphe python library', 'An access to the Onyphe API (apikey)'], - 'features': 'This module takes a domain, hostname, or IP address attribute as input in order to query the Onyphe API. Data fetched from the query is then parsed and MISP attributes are extracted.', - 'references': ['https://www.onyphe.io/', 'https://github.com/sebdraven/pyonyphe'], - 'input': 'A domain, hostname or IP address MISP attribute.', - 'output': 'MISP attributes fetched from the Onyphe query.', + "version": "2", + "author": "Sebastien Larinier @sebdraven", + "description": "Module to process a query on Onyphe.", + "module-type": ["expansion", "hover"], + "name": "Onyphe Lookup", + "logo": "onyphe.jpg", + "requirements": ["onyphe python library", "An access to the Onyphe API (apikey)"], + "features": ( + "This module takes a domain, hostname, or IP address attribute as input in order to query the Onyphe API. Data" + " fetched from the query is then parsed and MISP attributes are extracted." + ), + "references": ["https://www.onyphe.io/", "https://github.com/sebdraven/pyonyphe"], + "input": "A domain, hostname or IP address MISP attribute.", + "output": "MISP attributes fetched from the Onyphe query.", } # config fields that your code expects from the site admin -moduleconfig = ['apikey'] +moduleconfig = ["apikey"] class OnypheClient: @@ -43,174 +45,159 @@ def __init__(self, api_key, attribute): def get_results(self): event = json.loads(self.misp_event.to_json()) - results = {key: event[key] - for key in ('Attribute', 'Object') if key in event} + results = {key: event[key] for key in ("Attribute", "Object") if key in event} return results def get_query_onyphe(self): - if self.attribute['type'] == 'ip-src' or self.attribute['type'] == 'ip-dst': + if self.attribute["type"] == "ip-src" or self.attribute["type"] == "ip-dst": self.__summary_ip() - if self.attribute['type'] == 'domain': + if self.attribute["type"] == "domain": self.__summary_domain() - if self.attribute['type'] == 'hostname': + if self.attribute["type"] == "hostname": self.__summary_hostname() def __summary_ip(self): - results = self.onyphe_client.summary_ip(self.attribute['value']) - if 'results' in results: - for r in results['results']: - if 'domain' in r: - domain = r['domain'] - if type(domain) == list: + results = self.onyphe_client.summary_ip(self.attribute["value"]) + if "results" in results: + for r in results["results"]: + if "domain" in r: + domain = r["domain"] + if isinstance(domain, list): for d in domain: - self.__get_object_domain_ip(d, 'domain') - elif type(domain) == str: - self.__get_object_domain_ip(domain, 'domain') + self.__get_object_domain_ip(d, "domain") + elif isinstance(domain, str): + self.__get_object_domain_ip(domain, "domain") - if 'hostname' in r: - hostname = r['hostname'] - if type(hostname) == list: + if "hostname" in r: + hostname = r["hostname"] + if isinstance(hostname, list): for d in hostname: - self.__get_object_domain_ip(d, 'domain') - elif type(hostname) == str: - self.__get_object_domain_ip(hostname, 'domain') + self.__get_object_domain_ip(d, "domain") + elif isinstance(hostname, str): + self.__get_object_domain_ip(hostname, "domain") - if 'issuer' in r: + if "issuer" in r: self.__get_object_certificate(r) def __summary_domain(self): - results = self.onyphe_client.summary_domain(self.attribute['value']) - if 'results' in results: - for r in results['results']: - - for domain in r.get('domain'): - self.misp_event.add_attribute('domain', domain) - for hostname in r.get('hostname'): - self.misp_event.add_attribute('hostname', hostname) - if 'ip' in r: - if type(r['ip']) is str: - self.__get_object_domain_ip(r['ip'], 'ip') - if type(r['ip']) is list: - for ip in r['ip']: - self.__get_object_domain_ip(ip, 'ip') - if 'issuer' in r: + results = self.onyphe_client.summary_domain(self.attribute["value"]) + if "results" in results: + for r in results["results"]: + + for domain in r.get("domain"): + self.misp_event.add_attribute("domain", domain) + for hostname in r.get("hostname"): + self.misp_event.add_attribute("hostname", hostname) + if "ip" in r: + if type(r["ip"]) is str: + self.__get_object_domain_ip(r["ip"], "ip") + if type(r["ip"]) is list: + for ip in r["ip"]: + self.__get_object_domain_ip(ip, "ip") + if "issuer" in r: self.__get_object_certificate(r) def __summary_hostname(self): - results = self.onyphe_client.summary_hostname(self.attribute['value']) - if 'results' in results: - - for r in results['results']: - if 'domain' in r: - if type(r['domain']) is str: - self.misp_event.add_attribute( - 'domain', r['domain']) - if type(r['domain']) is list: - for domain in r['domain']: - self.misp_event.add_attribute('domain', domain) - - if 'hostname' in r: - if type(r['hostname']) is str: - self.misp_event.add_attribute( - 'hostname', r['hostname']) - if type(r['hostname']) is list: - for hostname in r['hostname']: - self.misp_event.add_attribute( - 'hostname', hostname) - - if 'ip' in r: - if type(r['ip']) is str: - self.__get_object_domain_ip(r['ip'], 'ip') - if type(r['ip']) is list: - for ip in r['ip']: - self.__get_object_domain_ip(ip, 'ip') - - if 'issuer' in r: + results = self.onyphe_client.summary_hostname(self.attribute["value"]) + if "results" in results: + + for r in results["results"]: + if "domain" in r: + if type(r["domain"]) is str: + self.misp_event.add_attribute("domain", r["domain"]) + if type(r["domain"]) is list: + for domain in r["domain"]: + self.misp_event.add_attribute("domain", domain) + + if "hostname" in r: + if type(r["hostname"]) is str: + self.misp_event.add_attribute("hostname", r["hostname"]) + if type(r["hostname"]) is list: + for hostname in r["hostname"]: + self.misp_event.add_attribute("hostname", hostname) + + if "ip" in r: + if type(r["ip"]) is str: + self.__get_object_domain_ip(r["ip"], "ip") + if type(r["ip"]) is list: + for ip in r["ip"]: + self.__get_object_domain_ip(ip, "ip") + + if "issuer" in r: self.__get_object_certificate(r) - if 'cve' in r: - if type(r['cve']) is list: - for cve in r['cve']: + if "cve" in r: + if type(r["cve"]) is list: + for cve in r["cve"]: self.__get_object_cve(r, cve) def __get_object_certificate(self, r): - object_certificate = MISPObject('x509') - object_certificate.add_attribute('ip', self.attribute['value']) - object_certificate.add_attribute('serial-number', r['serial']) - object_certificate.add_attribute( - 'x509-fingerprint-sha256', r['fingerprint']['sha256']) - object_certificate.add_attribute( - 'x509-fingerprint-sha1', r['fingerprint']['sha1']) - object_certificate.add_attribute( - 'x509-fingerprint-md5', r['fingerprint']['md5']) - - signature = r['signature']['algorithm'] - value = '' - if 'sha256' in signature and 'RSA' in signature: - value = 'SHA256_WITH_RSA_ENCRYPTION' - elif 'sha1' in signature and 'RSA' in signature: - value = 'SHA1_WITH_RSA_ENCRYPTION' + object_certificate = MISPObject("x509") + object_certificate.add_attribute("ip", self.attribute["value"]) + object_certificate.add_attribute("serial-number", r["serial"]) + object_certificate.add_attribute("x509-fingerprint-sha256", r["fingerprint"]["sha256"]) + object_certificate.add_attribute("x509-fingerprint-sha1", r["fingerprint"]["sha1"]) + object_certificate.add_attribute("x509-fingerprint-md5", r["fingerprint"]["md5"]) + + signature = r["signature"]["algorithm"] + value = "" + if "sha256" in signature and "RSA" in signature: + value = "SHA256_WITH_RSA_ENCRYPTION" + elif "sha1" in signature and "RSA" in signature: + value = "SHA1_WITH_RSA_ENCRYPTION" if value: - object_certificate.add_attribute('signature_algorithm', value) - - object_certificate.add_attribute( - 'pubkey-info-algorithm', r['publickey']['algorithm']) - - if 'exponent' in r['publickey']: - object_certificate.add_attribute( - 'pubkey-info-exponent', r['publickey']['exponent']) - if 'length' in r['publickey']: - object_certificate.add_attribute( - 'pubkey-info-size', r['publickey']['length']) - - object_certificate.add_attribute('issuer', r['issuer']['commonname']) - object_certificate.add_attribute( - 'validity-not-before', r['validity']['notbefore']) - object_certificate.add_attribute( - 'validity-not-after', r['validity']['notbefore']) - object_certificate.add_reference(self.attribute['uuid'], 'related-to') + object_certificate.add_attribute("signature_algorithm", value) + + object_certificate.add_attribute("pubkey-info-algorithm", r["publickey"]["algorithm"]) + + if "exponent" in r["publickey"]: + object_certificate.add_attribute("pubkey-info-exponent", r["publickey"]["exponent"]) + if "length" in r["publickey"]: + object_certificate.add_attribute("pubkey-info-size", r["publickey"]["length"]) + + object_certificate.add_attribute("issuer", r["issuer"]["commonname"]) + object_certificate.add_attribute("validity-not-before", r["validity"]["notbefore"]) + object_certificate.add_attribute("validity-not-after", r["validity"]["notbefore"]) + object_certificate.add_reference(self.attribute["uuid"], "related-to") self.misp_event.add_object(object_certificate) def __get_object_domain_ip(self, obs, relation): - objet_domain_ip = MISPObject('domain-ip') + objet_domain_ip = MISPObject("domain-ip") objet_domain_ip.add_attribute(relation, obs) relation_attr = self.__get_relation_attribute() if relation_attr: - objet_domain_ip.add_attribute( - relation_attr, self.attribute['value']) - objet_domain_ip.add_reference(self.attribute['uuid'], 'related-to') + objet_domain_ip.add_attribute(relation_attr, self.attribute["value"]) + objet_domain_ip.add_reference(self.attribute["uuid"], "related-to") self.misp_event.add_object(objet_domain_ip) def __get_relation_attribute(self): - if self.attribute['type'] == 'ip-src': - return 'ip' - elif self.attribute['type'] == 'ip-dst': - return 'ip' - elif self.attribute['type'] == 'domain': - return 'domain' - elif self.attribute['type'] == 'hostname': - return 'domain' + if self.attribute["type"] == "ip-src": + return "ip" + elif self.attribute["type"] == "ip-dst": + return "ip" + elif self.attribute["type"] == "domain": + return "domain" + elif self.attribute["type"] == "hostname": + return "domain" def __get_object_cve(self, item, cve): attributes = [] - object_cve = MISPObject('vulnerability') - object_cve.add_attribute('id', cve) - object_cve.add_attribute('state', 'Published') - - if type(item['ip']) is list: - for ip in item['ip']: - attributes.extend( - list(filter(lambda x: x['value'] == ip, self.misp_event['Attribute']))) - for obj in self.misp_event['Object']: - attributes.extend( - list(filter(lambda x: x['value'] == ip, obj['Attribute']))) - if type(item['ip']) is str: - - for obj in self.misp_event['Object']: - for att in obj['Attribute']: - if att['value'] == item['ip']: - object_cve.add_reference(obj['uuid'], 'cve') + object_cve = MISPObject("vulnerability") + object_cve.add_attribute("id", cve) + object_cve.add_attribute("state", "Published") + + if type(item["ip"]) is list: + for ip in item["ip"]: + attributes.extend(list(filter(lambda x: x["value"] == ip, self.misp_event["Attribute"]))) + for obj in self.misp_event["Object"]: + attributes.extend(list(filter(lambda x: x["value"] == ip, obj["Attribute"]))) + if type(item["ip"]) is str: + + for obj in self.misp_event["Object"]: + for att in obj["Attribute"]: + if att["value"] == item["ip"]: + object_cve.add_reference(obj["uuid"], "cve") self.misp_event.add_object(object_cve) @@ -219,19 +206,19 @@ def handler(q=False): if q: request = json.loads(q) - attribute = request['attribute'] + attribute = request["attribute"] - if not request.get('config') or not request['config'].get('apikey'): - misperrors['error'] = 'Onyphe authentication is missing' + if not request.get("config") or not request["config"].get("apikey"): + misperrors["error"] = "Onyphe authentication is missing" return misperrors - api_key = request['config'].get('apikey') + api_key = request["config"].get("apikey") onyphe_client = OnypheClient(api_key, attribute) onyphe_client.get_query_onyphe() results = onyphe_client.get_results() - return {'results': results} + return {"results": results} def introspection(): @@ -239,5 +226,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index 417d751d6..b4d9eaf29 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -1,33 +1,38 @@ # -*- coding: utf-8 -*- import json -try: - from onyphe import Onyphe -except ImportError: - print("pyonyphe module not installed.") -misperrors = {'error': 'Error'} +from onyphe import Onyphe -mispattributes = {'input': ['ip-src', 'ip-dst', 'hostname', 'domain'], - 'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'url']} +misperrors = {"error": "Error"} + +mispattributes = { + "input": ["ip-src", "ip-dst", "hostname", "domain"], + "output": ["hostname", "domain", "ip-src", "ip-dst", "url"], +} # possible module-types: 'expansion', 'hover' or both moduleinfo = { - 'version': '1', - 'author': 'Sebastien Larinier @sebdraven', - 'description': 'Module to process a full query on Onyphe.', - 'module-type': ['expansion', 'hover'], - 'name': 'Onyphe Full Lookup', - 'logo': 'onyphe.jpg', - 'requirements': ['onyphe python library', 'An access to the Onyphe API (apikey)'], - 'features': 'This module takes a domain, hostname, or IP address attribute as input in order to query the Onyphe API. Data fetched from the query is then parsed and MISP attributes are extracted.\n\nThe parsing is here more advanced than the one on onyphe module, and is returning more attributes, since more fields of the query result are watched and parsed.', - 'references': ['https://www.onyphe.io/', 'https://github.com/sebdraven/pyonyphe'], - 'input': 'A domain, hostname or IP address MISP attribute.', - 'output': 'MISP attributes fetched from the Onyphe query.', + "version": "1", + "author": "Sebastien Larinier @sebdraven", + "description": "Module to process a full query on Onyphe.", + "module-type": ["expansion", "hover"], + "name": "Onyphe Full Lookup", + "logo": "onyphe.jpg", + "requirements": ["onyphe python library", "An access to the Onyphe API (apikey)"], + "features": ( + "This module takes a domain, hostname, or IP address attribute as input in order to query the Onyphe API. Data" + " fetched from the query is then parsed and MISP attributes are extracted.\n\nThe parsing is here more advanced" + " than the one on onyphe module, and is returning more attributes, since more fields of the query result are" + " watched and parsed." + ), + "references": ["https://www.onyphe.io/", "https://github.com/sebdraven/pyonyphe"], + "input": "A domain, hostname or IP address MISP attribute.", + "output": "MISP attributes fetched from the Onyphe query.", } # config fields that your code expects from the site admin -moduleconfig = ['apikey'] +moduleconfig = ["apikey"] def handler(q=False): @@ -35,30 +40,30 @@ def handler(q=False): request = json.loads(q) - if not request.get('config') or not request['config'].get('apikey'): - misperrors['error'] = 'Onyphe authentication is missing' + if not request.get("config") or not request["config"].get("apikey"): + misperrors["error"] = "Onyphe authentication is missing" return misperrors - api = Onyphe(request['config'].get('apikey')) + api = Onyphe(request["config"].get("apikey")) if not api: - misperrors['error'] = 'Onyphe Error instance api' + misperrors["error"] = "Onyphe Error instance api" - ip = '' - if request.get('ip-src'): - ip = request['ip-src'] + ip = "" + if request.get("ip-src"): + ip = request["ip-src"] return handle_ip(api, ip, misperrors) - elif request.get('ip-dst'): - ip = request['ip-dst'] + elif request.get("ip-dst"): + ip = request["ip-dst"] return handle_ip(api, ip, misperrors) - elif request.get('domain'): - domain = request['domain'] + elif request.get("domain"): + domain = request["domain"] return handle_domain(api, domain, misperrors) - elif request.get('hostname'): - hostname = request['hostname'] + elif request.get("hostname"): + hostname = request["hostname"] return handle_domain(api, hostname, misperrors) else: - misperrors['error'] = "Unsupported attributes type" + misperrors["error"] = "Unsupported attributes type" return misperrors else: return False @@ -70,25 +75,25 @@ def handle_domain(api, domain, misperrors): r, status_ok = expand_pastries(api, misperrors, domain=domain) if status_ok: - result_filtered['results'].extend(r) + result_filtered["results"].extend(r) else: - misperrors['error'] = 'Error pastries result' + misperrors["error"] = "Error pastries result" return misperrors r, status_ok = expand_datascan(api, misperrors, domain=domain) if status_ok: - result_filtered['results'].extend(r) + result_filtered["results"].extend(r) else: - misperrors['error'] = 'Error datascan result ' + misperrors["error"] = "Error datascan result " return misperrors r, status_ok = expand_threatlist(api, misperrors, domain=domain) if status_ok: - result_filtered['results'].extend(r) + result_filtered["results"].extend(r) else: - misperrors['error'] = 'Error threat list' + misperrors["error"] = "Error threat list" return misperrors return result_filtered @@ -100,48 +105,48 @@ def handle_ip(api, ip, misperrors): r, status_ok = expand_syscan(api, ip, misperrors) if status_ok: - result_filtered['results'].extend(r) + result_filtered["results"].extend(r) else: - misperrors['error'] = "Error syscan result" + misperrors["error"] = "Error syscan result" r, status_ok = expand_pastries(api, misperrors, ip=ip) if status_ok: - result_filtered['results'].extend(r) + result_filtered["results"].extend(r) else: - misperrors['error'] = 'Error pastries result' + misperrors["error"] = "Error pastries result" return misperrors r, status_ok = expand_datascan(api, misperrors, ip=ip) if status_ok: - result_filtered['results'].extend(r) + result_filtered["results"].extend(r) else: - misperrors['error'] = 'Error datascan result ' + misperrors["error"] = "Error datascan result " return misperrors r, status_ok = expand_forward(api, ip, misperrors) if status_ok: - result_filtered['results'].extend(r) + result_filtered["results"].extend(r) else: - misperrors['error'] = 'Error forward result' + misperrors["error"] = "Error forward result" return misperrors r, status_ok = expand_reverse(api, ip, misperrors) if status_ok: - result_filtered['results'].extend(r) + result_filtered["results"].extend(r) else: - misperrors['error'] = 'Error reverse result' + misperrors["error"] = "Error reverse result" return misperrors r, status_ok = expand_threatlist(api, misperrors, ip=ip) if status_ok: - result_filtered['results'].extend(r) + result_filtered["results"].extend(r) else: - misperrors['error'] = 'Error threat list' + misperrors["error"] = "Error threat list" return misperrors return result_filtered @@ -156,40 +161,51 @@ def expand_syscan(api, ip, misperror): orgs = [] results = api.synscan(ip) - if results['status'] == 'ok': + if results["status"] == "ok": status_ok = True - for elem in results['results']: - asn_list.append(elem['asn']) - os_target = elem['os'] - geoloc.append(elem['location']) - orgs.append(elem['organization']) - if os_target != 'Unknown' and os_target != 'Undefined': - os_list.append(elem['os']) - - r.append({'types': ['target-machine'], - 'values': list(set(os_list)), - 'categories': ['Targeting data'], - 'comment': 'OS found on %s with synscan of Onyphe' % ip}) - - r.append({'types': ['target-location'], - 'values': list(set(geoloc)), - 'categories': ['Targeting data'], - 'comment': 'Geolocalisation of %s found with synscan of Onyphe' - % ip - }) - - r.append({'types': ['target-org'], - 'values': list(set(orgs)), - 'categories': ['Targeting data'], - 'comment': 'Organisations of %s found with synscan of Onyphe' - % ip - }) - - r.append({'types': ['AS'], - 'values': list(set(asn_list)), - 'categories': ['Network activity'], - 'comment': 'As number of %s found with synscan of Onyphe' % ip - }) + for elem in results["results"]: + asn_list.append(elem["asn"]) + os_target = elem["os"] + geoloc.append(elem["location"]) + orgs.append(elem["organization"]) + if os_target != "Unknown" and os_target != "Undefined": + os_list.append(elem["os"]) + + r.append( + { + "types": ["target-machine"], + "values": list(set(os_list)), + "categories": ["Targeting data"], + "comment": "OS found on %s with synscan of Onyphe" % ip, + } + ) + + r.append( + { + "types": ["target-location"], + "values": list(set(geoloc)), + "categories": ["Targeting data"], + "comment": "Geolocalisation of %s found with synscan of Onyphe" % ip, + } + ) + + r.append( + { + "types": ["target-org"], + "values": list(set(orgs)), + "categories": ["Targeting data"], + "comment": "Organisations of %s found with synscan of Onyphe" % ip, + } + ) + + r.append( + { + "types": ["AS"], + "values": list(set(asn_list)), + "categories": ["Network activity"], + "comment": "As number of %s found with synscan of Onyphe" % ip, + } + ) return r, status_ok @@ -198,53 +214,62 @@ def expand_datascan(api, misperror, **kwargs): status_ok = False r = [] # ip = '' - query = '' + query = "" asn_list = [] geoloc = [] orgs = [] ports = [] - if 'ip' in kwargs: - query = kwargs.get('ip') + if "ip" in kwargs: + query = kwargs.get("ip") results = api.datascan(query) else: - query = kwargs.get('domain') - results = api.search_datascan('domain:%s' % query) + query = kwargs.get("domain") + results = api.search_datascan("domain:%s" % query) - if results['status'] == 'ok': + if results["status"] == "ok": status_ok = True - for elem in results['results']: - asn_list.append(elem['asn']) - geoloc.append(elem['location']) - orgs.append(elem['organization']) - ports.append(elem['port']) - - r.append({'types': ['port'], - 'values': list(set(ports)), - 'categories': ['Other'], - 'comment': 'Ports of %s found with datascan of Onyphe' - % query - }) - - r.append({'types': ['target-location'], - 'values': list(set(geoloc)), - 'categories': ['Targeting data'], - 'comment': 'Geolocalisation of %s found with synscan of Onyphe' - % query - }) - - r.append({'types': ['target-org'], - 'values': list(set(orgs)), - 'categories': ['Targeting data'], - 'comment': 'Organisations of %s found with synscan of Onyphe' - % query - }) - - r.append({'types': ['AS'], - 'values': list(set(asn_list)), - 'categories': ['Network activity'], - 'comment': 'As number of %s found with synscan of Onyphe' % query - }) + for elem in results["results"]: + asn_list.append(elem["asn"]) + geoloc.append(elem["location"]) + orgs.append(elem["organization"]) + ports.append(elem["port"]) + + r.append( + { + "types": ["port"], + "values": list(set(ports)), + "categories": ["Other"], + "comment": "Ports of %s found with datascan of Onyphe" % query, + } + ) + + r.append( + { + "types": ["target-location"], + "values": list(set(geoloc)), + "categories": ["Targeting data"], + "comment": "Geolocalisation of %s found with synscan of Onyphe" % query, + } + ) + + r.append( + { + "types": ["target-org"], + "values": list(set(orgs)), + "categories": ["Targeting data"], + "comment": "Organisations of %s found with synscan of Onyphe" % query, + } + ) + + r.append( + { + "types": ["AS"], + "values": list(set(asn_list)), + "categories": ["Network activity"], + "comment": "As number of %s found with synscan of Onyphe" % query, + } + ) return r, status_ok @@ -258,22 +283,30 @@ def expand_reverse(api, ip, misperror): domains_reverse = [] domains = [] - if results['status'] == 'ok': + if results["status"] == "ok": status_ok = True - for elem in results['results']: - domains_reverse.append(elem['reverse']) - domains.append(elem['domain']) - - r.append({'types': ['domain'], - 'values': list(set(domains)), - 'categories': ['Network activity'], - 'comment': 'Domains of %s from forward service of Onyphe' % ip}) - - r.append({'types': ['domain'], - 'values': list(set(domains_reverse)), - 'categories': ['Network activity'], - 'comment': 'Reverse Domains of %s from forward service of Onyphe' % ip}) + for elem in results["results"]: + domains_reverse.append(elem["reverse"]) + domains.append(elem["domain"]) + + r.append( + { + "types": ["domain"], + "values": list(set(domains)), + "categories": ["Network activity"], + "comment": "Domains of %s from forward service of Onyphe" % ip, + } + ) + + r.append( + { + "types": ["domain"], + "values": list(set(domains_reverse)), + "categories": ["Network activity"], + "comment": "Reverse Domains of %s from forward service of Onyphe" % ip, + } + ) return r, status_ok @@ -285,22 +318,30 @@ def expand_forward(api, ip, misperror): domains_forward = [] domains = [] - if results['status'] == 'ok': + if results["status"] == "ok": status_ok = True - for elem in results['results']: - domains_forward.append(elem['forward']) - domains.append(elem['domain']) - - r.append({'types': ['domain'], - 'values': list(set(domains)), - 'categories': ['Network activity'], - 'comment': 'Domains of %s from forward service of Onyphe' % ip}) - - r.append({'types': ['domain'], - 'values': list(set(domains_forward)), - 'categories': ['Network activity'], - 'comment': 'Forward Domains of %s from forward service of Onyphe' % ip}) + for elem in results["results"]: + domains_forward.append(elem["forward"]) + domains.append(elem["domain"]) + + r.append( + { + "types": ["domain"], + "values": list(set(domains)), + "categories": ["Network activity"], + "comment": "Domains of %s from forward service of Onyphe" % ip, + } + ) + + r.append( + { + "types": ["domain"], + "values": list(set(domains_forward)), + "categories": ["Network activity"], + "comment": "Forward Domains of %s from forward service of Onyphe" % ip, + } + ) return r, status_ok @@ -313,38 +354,52 @@ def expand_pastries(api, misperror, **kwargs): urls_pasties = [] domains = [] ips = [] - if 'ip' in kwargs: - query = kwargs.get('ip') + if "ip" in kwargs: + query = kwargs.get("ip") result = api.pastries(query) - if 'domain' in kwargs: - query = kwargs.get('domain') - result = api.search_pastries('domain:%s' % query) + if "domain" in kwargs: + query = kwargs.get("domain") + result = api.search_pastries("domain:%s" % query) - if result['status'] == 'ok': + if result["status"] == "ok": status_ok = True - for item in result['results']: - if item['@category'] == 'pastries': - if item['source'] == 'pastebin': - urls_pasties.append('https://pastebin.com/raw/%s' % item['key']) - - if 'domain' in item: - domains.extend(item['domain']) - if 'ip' in item: - ips.extend(item['ip']) - if 'hostname' in item: - domains.extend(item['hostname']) - - r.append({'types': ['url'], - 'values': urls_pasties, - 'categories': ['External analysis'], - 'comment': 'URLs of pasties where %s has found' % query}) - r.append({'types': ['domain'], 'values': list(set(domains)), - 'categories': ['Network activity'], - 'comment': 'Domains found in pasties of Onyphe'}) - - r.append({'types': ['ip-dst'], 'values': list(set(ips)), - 'categories': ['Network activity'], - 'comment': 'IPs found in pasties of Onyphe'}) + for item in result["results"]: + if item["@category"] == "pastries": + if item["source"] == "pastebin": + urls_pasties.append("https://pastebin.com/raw/%s" % item["key"]) + + if "domain" in item: + domains.extend(item["domain"]) + if "ip" in item: + ips.extend(item["ip"]) + if "hostname" in item: + domains.extend(item["hostname"]) + + r.append( + { + "types": ["url"], + "values": urls_pasties, + "categories": ["External analysis"], + "comment": "URLs of pasties where %s has found" % query, + } + ) + r.append( + { + "types": ["domain"], + "values": list(set(domains)), + "categories": ["Network activity"], + "comment": "Domains found in pasties of Onyphe", + } + ) + + r.append( + { + "types": ["ip-dst"], + "values": list(set(ips)), + "categories": ["Network activity"], + "comment": "IPs found in pasties of Onyphe", + } + ) return r, status_ok @@ -357,23 +412,25 @@ def expand_threatlist(api, misperror, **kwargs): threat_list = [] - if 'ip' in kwargs: - query = kwargs.get('ip') + if "ip" in kwargs: + query = kwargs.get("ip") results = api.threatlist(query) else: - query = kwargs.get('domain') - results = api.search_threatlist('domain:%s' % query) + query = kwargs.get("domain") + results = api.search_threatlist("domain:%s" % query) - if results['status'] == 'ok': + if results["status"] == "ok": status_ok = True - threat_list = ['seen %s on %s ' % (item['seen_date'], item['threatlist']) - for item in results['results']] + threat_list = ["seen %s on %s " % (item["seen_date"], item["threatlist"]) for item in results["results"]] - r.append({'types': ['comment'], - 'categories': ['Other'], - 'values': threat_list, - 'comment': '%s is present in threatlist' % query - }) + r.append( + { + "types": ["comment"], + "categories": ["Other"], + "values": threat_list, + "comment": "%s is present in threatlist" % query, + } + ) return r, status_ok @@ -383,5 +440,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/otx.py b/misp_modules/modules/expansion/otx.py index 97c169fcb..7630ff28c 100755 --- a/misp_modules/modules/expansion/otx.py +++ b/misp_modules/modules/expansion/otx.py @@ -1,25 +1,55 @@ import json -import requests import re -misperrors = {'error': 'Error'} -mispattributes = {'input': ["hostname", "domain", "ip-src", "ip-dst", "md5", "sha1", "sha256", "sha512"], - 'output': ["domain", "ip-src", "ip-dst", "text", "md5", "sha1", "sha256", "sha512", "email"] - } +import requests + +misperrors = {"error": "Error"} +mispattributes = { + "input": [ + "hostname", + "domain", + "ip-src", + "ip-dst", + "md5", + "sha1", + "sha256", + "sha512", + ], + "output": [ + "domain", + "ip-src", + "ip-dst", + "text", + "md5", + "sha1", + "sha256", + "sha512", + "email", + ], +} # possible module-types: 'expansion', 'hover' or both moduleinfo = { - 'version': '1', - 'author': 'chrisdoman', - 'description': 'Module to get information from AlienVault OTX.', - 'module-type': ['expansion'], - 'name': 'AlienVault OTX Lookup', - 'logo': 'otx.png', - 'requirements': ['An access to the OTX API (apikey)'], - 'features': 'This module takes a MISP attribute as input to query the OTX Alienvault API. The API returns then the result of the query with some types we map into compatible types we add as MISP attributes.', - 'references': ['https://www.alienvault.com/open-threat-exchange'], - 'input': 'A MISP attribute included in the following list:\n- hostname\n- domain\n- ip-src\n- ip-dst\n- md5\n- sha1\n- sha256\n- sha512', - 'output': 'MISP attributes mapped from the result of the query on OTX, included in the following list:\n- domain\n- ip-src\n- ip-dst\n- text\n- md5\n- sha1\n- sha256\n- sha512\n- email', + "version": "1", + "author": "chrisdoman", + "description": "Module to get information from AlienVault OTX.", + "module-type": ["expansion"], + "name": "AlienVault OTX Lookup", + "logo": "otx.png", + "requirements": ["An access to the OTX API (apikey)"], + "features": ( + "This module takes a MISP attribute as input to query the OTX Alienvault API. The API returns then the result" + " of the query with some types we map into compatible types we add as MISP attributes." + ), + "references": ["https://www.alienvault.com/open-threat-exchange"], + "input": ( + "A MISP attribute included in the following list:\n- hostname\n- domain\n- ip-src\n- ip-dst\n- md5\n- sha1\n-" + " sha256\n- sha512" + ), + "output": ( + "MISP attributes mapped from the result of the query on OTX, included in the following list:\n- domain\n-" + " ip-src\n- ip-dst\n- text\n- md5\n- sha1\n- sha256\n- sha512\n- email" + ), } # We're not actually using the API key yet @@ -28,7 +58,13 @@ # Avoid adding windows update to enrichment etc. def isBlacklisted(value): - blacklist = ['0.0.0.0', '8.8.8.8', '255.255.255.255', '192.168.56.', 'time.windows.com'] + blacklist = [ + "0.0.0.0", + "8.8.8.8", + "255.255.255.255", + "192.168.56.", + "time.windows.com", + ] for b in blacklist: if value in b: @@ -58,7 +94,12 @@ def findAll(data, keys): def valid_email(email): - return bool(re.search(r"[a-zA-Z0-9!#$%&'*+\/=?^_`{|}~-]+(?:\.[a-zA-Z0-9!#$%&'*+\/=?^_`{|}~-]+)*@(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?\.)+[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?", email)) + return bool( + re.search( + r"[a-zA-Z0-9!#$%&'*+\/=?^_`{|}~-]+(?:\.[a-zA-Z0-9!#$%&'*+\/=?^_`{|}~-]+)*@(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?\.)+[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?", + email, + ) + ) def handler(q=False): @@ -77,16 +118,16 @@ def handler(q=False): r["results"] += getIP(q["ip-dst"], key) if "domain" in q: r["results"] += getDomain(q["domain"], key) - if 'hostname' in q: - r["results"] += getDomain(q['hostname'], key) - if 'md5' in q: - r["results"] += getHash(q['md5'], key) - if 'sha1' in q: - r["results"] += getHash(q['sha1'], key) - if 'sha256' in q: - r["results"] += getHash(q['sha256'], key) - if 'sha512' in q: - r["results"] += getHash(q['sha512'], key) + if "hostname" in q: + r["results"] += getDomain(q["hostname"], key) + if "md5" in q: + r["results"] += getHash(q["md5"], key) + if "sha1" in q: + r["results"] += getHash(q["sha1"], key) + if "sha256" in q: + r["results"] += getHash(q["sha256"], key) + if "sha512" in q: + r["results"] += getHash(q["sha512"], key) uniq = [] for res in r["results"]: @@ -132,7 +173,9 @@ def getDomain(domain, key): ret = [] - req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/domain/malware/" + domain + "?limit=1000").text) + req = json.loads( + requests.get("https://otx.alienvault.com/otxapi/indicator/domain/malware/" + domain + "?limit=1000").text + ) for _hash in findAll(req, "hash"): ret.append({"types": ["sha256"], "values": [_hash]}) @@ -163,5 +206,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/passive_ssh.py b/misp_modules/modules/expansion/passive_ssh.py index 2a3175ab2..bf2fa7ed4 100644 --- a/misp_modules/modules/expansion/passive_ssh.py +++ b/misp_modules/modules/expansion/passive_ssh.py @@ -1,37 +1,43 @@ import json -import requests -from . import check_input_attribute, standard_error_message from collections import defaultdict + +import requests from pymisp import MISPEvent, MISPObject -misperrors = {'error': 'Error'} +from . import check_input_attribute, standard_error_message + +misperrors = {"error": "Error"} -mispattributes = {'input': ['ip-src', 'ip-dst', 'ssh-fingerprint'], - 'format': 'misp_standard'} +mispattributes = { + "input": ["ip-src", "ip-dst", "ssh-fingerprint"], + "format": "misp_standard", +} moduleinfo = { - 'version': '1', - 'author': 'Jean-Louis Huynen', - 'description': 'An expansion module to enrich, SSH key fingerprints and IP addresses with information collected by passive-ssh', - 'module-type': ['expansion', 'hover'], - 'name': 'Passive SSH Enrichment', - 'logo': '', - 'requirements': [], - 'features': '', - 'references': [], - 'input': '', - 'output': '', + "version": "1", + "author": "Jean-Louis Huynen", + "description": ( + "An expansion module to enrich, SSH key fingerprints and IP addresses with information collected by passive-ssh" + ), + "module-type": ["expansion", "hover"], + "name": "Passive SSH Enrichment", + "logo": "", + "requirements": [], + "features": "", + "references": [], + "input": "", + "output": "", } moduleconfig = ["custom_api_url", "api_user", "api_key"] -passivessh_url = 'https://passivessh.circl.lu/' +passivessh_url = "https://passivessh.circl.lu/" -host_query = '/host/ssh' -fingerprint_query = '/fingerprint/all' +host_query = "/host/ssh" +fingerprint_query = "/fingerprint/all" -class PassivesshParser(): +class PassivesshParser: def __init__(self, attribute, passivesshresult): self.attribute = attribute self.passivesshresult = passivesshresult @@ -43,31 +49,34 @@ def get_result(self): if self.references: self.__build_references() event = json.loads(self.misp_event.to_json()) - results = {key: event[key] for key in ( - 'Attribute', 'Object') if (key in event and event[key])} - return {'results': results} + results = {key: event[key] for key in ("Attribute", "Object") if (key in event and event[key])} + return {"results": results} def parse_passivessh_information(self): - passivessh_object = MISPObject('passive-ssh') - if 'first_seen' in self.passivesshresult: - passivessh_object.add_attribute( - 'first_seen', **{'type': 'datetime', 'value': self.passivesshresult['first_seen']}) - if 'last_seen' in self.passivesshresult: + passivessh_object = MISPObject("passive-ssh") + if "first_seen" in self.passivesshresult: passivessh_object.add_attribute( - 'last_seen', **{'type': 'datetime', 'value': self.passivesshresult['last_seen']}) - if 'base64' in self.passivesshresult: + "first_seen", + **{"type": "datetime", "value": self.passivesshresult["first_seen"]}, + ) + if "last_seen" in self.passivesshresult: passivessh_object.add_attribute( - 'base64', **{'type': 'text', 'value': self.passivesshresult['base64']}) - if 'keys' in self.passivesshresult: - for key in self.passivesshresult['keys']: + "last_seen", + **{"type": "datetime", "value": self.passivesshresult["last_seen"]}, + ) + if "base64" in self.passivesshresult: + passivessh_object.add_attribute("base64", **{"type": "text", "value": self.passivesshresult["base64"]}) + if "keys" in self.passivesshresult: + for key in self.passivesshresult["keys"]: passivessh_object.add_attribute( - 'fingerprint', **{'type': 'ssh-fingerprint', 'value': key['fingerprint']}) - if 'hosts' in self.passivesshresult: - for host in self.passivesshresult['hosts']: - passivessh_object.add_attribute( - 'host', **{'type': 'ip-dst', 'value': host}) - - passivessh_object.add_reference(self.attribute['uuid'], 'related-to') + "fingerprint", + **{"type": "ssh-fingerprint", "value": key["fingerprint"]}, + ) + if "hosts" in self.passivesshresult: + for host in self.passivesshresult["hosts"]: + passivessh_object.add_attribute("host", **{"type": "ip-dst", "value": host}) + + passivessh_object.add_reference(self.attribute["uuid"], "related-to") self.misp_event.add_object(passivessh_object) def __build_references(self): @@ -80,7 +89,7 @@ def __build_references(self): def check_url(url): - return "{}/".format(url) if not url.endswith('/') else url + return "{}/".format(url) if not url.endswith("/") else url def handler(q=False): @@ -89,49 +98,49 @@ def handler(q=False): return False request = json.loads(q) - api_url = check_url(request['config']['custom_api_url']) if request['config'].get( - 'custom_api_url') else passivessh_url + api_url = ( + check_url(request["config"]["custom_api_url"]) if request["config"].get("custom_api_url") else passivessh_url + ) - if request['config'].get('api_user'): - api_user = request['config'].get('api_user') + if request["config"].get("api_user"): + api_user = request["config"].get("api_user") else: - misperrors['error'] = 'passive-ssh user required' + misperrors["error"] = "passive-ssh user required" return misperrors - if request['config'].get('api_key'): - api_key = request['config'].get('api_key') + if request["config"].get("api_key"): + api_key = request["config"].get("api_key") else: - misperrors['error'] = 'passive-ssh password required' + misperrors["error"] = "passive-ssh password required" return misperrors - if not request.get('attribute') or not check_input_attribute(request['attribute']): - return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} - attribute = request['attribute'] - if attribute.get('type') == 'ip-src': + if not request.get("attribute") or not check_input_attribute(request["attribute"]): + return {"error": f"{standard_error_message}, which should contain at least a type, a value and an uuid."} + attribute = request["attribute"] + if attribute.get("type") == "ip-src": type = host_query pass - elif attribute.get('type') == 'ip-dst': + elif attribute.get("type") == "ip-dst": type = host_query pass - elif attribute.get('type') == 'ssh-fingerprint': + elif attribute.get("type") == "ssh-fingerprint": type = fingerprint_query pass else: - misperrors['error'] = 'ip is missing.' + misperrors["error"] = "ip is missing." return misperrors - r = requests.get("{}{}/{}".format(api_url, type, - attribute['value']), auth=(api_user, api_key)) + r = requests.get("{}{}/{}".format(api_url, type, attribute["value"]), auth=(api_user, api_key)) if r.status_code == 200: passivesshresult = r.json() if not passivesshresult: - misperrors['error'] = 'Empty result' + misperrors["error"] = "Empty result" return misperrors elif r.status_code == 404: - misperrors['error'] = 'Non existing hash' + misperrors["error"] = "Non existing hash" return misperrors else: - misperrors['error'] = 'API not accessible' + misperrors["error"] = "API not accessible" return misperrors parser = PassivesshParser(attribute, passivesshresult) @@ -146,5 +155,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/passivetotal.py b/misp_modules/modules/expansion/passivetotal.py index 679d434c3..deb99f223 100755 --- a/misp_modules/modules/expansion/passivetotal.py +++ b/misp_modules/modules/expansion/passivetotal.py @@ -4,89 +4,142 @@ from passivetotal.common.utilities import is_ip - -log = logging.getLogger('passivetotal') +log = logging.getLogger("passivetotal") log.setLevel(logging.DEBUG) ch = logging.StreamHandler(sys.stdout) ch.setLevel(logging.DEBUG) -formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") ch.setFormatter(formatter) log.addHandler(ch) -misperrors = {'error': 'Error'} +misperrors = {"error": "Error"} mispattributes = { - 'input': ['hostname', 'domain', 'ip-src', 'ip-dst', - 'x509-fingerprint-sha1', 'email-src', 'email-dst', - 'target-email', 'whois-registrant-email', - 'whois-registrant-phone', 'text', 'whois-registrant-name', - 'whois-registrar', 'whois-creation-date'], - 'output': ['hostname', 'domain', 'ip-src', 'ip-dst', - 'x509-fingerprint-sha1', 'email-src', 'email-dst', - 'target-email', 'whois-registrant-email', - 'whois-registrant-phone', 'text', 'whois-registrant-name', - 'whois-registrar', 'whois-creation-date', 'md5', 'sha1', - 'sha256', 'link'] + "input": [ + "hostname", + "domain", + "ip-src", + "ip-dst", + "x509-fingerprint-sha1", + "email-src", + "email-dst", + "target-email", + "whois-registrant-email", + "whois-registrant-phone", + "text", + "whois-registrant-name", + "whois-registrar", + "whois-creation-date", + ], + "output": [ + "hostname", + "domain", + "ip-src", + "ip-dst", + "x509-fingerprint-sha1", + "email-src", + "email-dst", + "target-email", + "whois-registrant-email", + "whois-registrant-phone", + "text", + "whois-registrant-name", + "whois-registrar", + "whois-creation-date", + "md5", + "sha1", + "sha256", + "link", + ], } moduleinfo = { - 'version': '1.0', - 'author': 'Brandon Dixon', - 'description': 'The PassiveTotal MISP expansion module brings the datasets derived from Internet scanning directly into your MISP instance. This module supports passive DNS, historic SSL, WHOIS, and host attributes. In order to use the module, you must have a valid PassiveTotal account username and API key. Registration is free and can be done by visiting https://www.passivetotal.org/register', - 'module-type': ['expansion', 'hover'], - 'name': 'PassiveTotal Lookup', - 'logo': 'passivetotal.png', - 'requirements': ['Passivetotal python library', 'An access to the PassiveTotal API (apikey)'], - 'features': 'The PassiveTotal MISP expansion module brings the datasets derived from Internet scanning directly into your MISP instance. This module supports passive DNS, historic SSL, WHOIS, and host attributes. In order to use the module, you must have a valid PassiveTotal account username and API key. Registration is free and can be done by visiting https://www.passivetotal.org/register', - 'references': ['https://www.passivetotal.org/register'], - 'input': 'A MISP attribute included in the following list:\n- hostname\n- domain\n- ip-src\n- ip-dst\n- x509-fingerprint-sha1\n- email-src\n- email-dst\n- target-email\n- whois-registrant-email\n- whois-registrant-phone\n- text\n- whois-registrant-name\n- whois-registrar\n- whois-creation-date', - 'output': 'MISP attributes mapped from the result of the query on PassiveTotal, included in the following list:\n- hostname\n- domain\n- ip-src\n- ip-dst\n- x509-fingerprint-sha1\n- email-src\n- email-dst\n- target-email\n- whois-registrant-email\n- whois-registrant-phone\n- text\n- whois-registrant-name\n- whois-registrar\n- whois-creation-date\n- md5\n- sha1\n- sha256\n- link', + "version": "1.0", + "author": "Brandon Dixon", + "description": ( + "The PassiveTotal MISP expansion module brings the datasets derived from Internet scanning directly into your" + " MISP instance. This module supports passive DNS, historic SSL, WHOIS, and host attributes. In order to use" + " the module, you must have a valid PassiveTotal account username and API key. Registration is free and can be" + " done by visiting https://www.passivetotal.org/register" + ), + "module-type": ["expansion", "hover"], + "name": "PassiveTotal Lookup", + "logo": "passivetotal.png", + "requirements": [ + "Passivetotal python library", + "An access to the PassiveTotal API (apikey)", + ], + "features": ( + "The PassiveTotal MISP expansion module brings the datasets derived from Internet scanning directly into your" + " MISP instance. This module supports passive DNS, historic SSL, WHOIS, and host attributes. In order to use" + " the module, you must have a valid PassiveTotal account username and API key. Registration is free and can be" + " done by visiting https://www.passivetotal.org/register" + ), + "references": ["https://www.passivetotal.org/register"], + "input": ( + "A MISP attribute included in the following list:\n- hostname\n- domain\n- ip-src\n- ip-dst\n-" + " x509-fingerprint-sha1\n- email-src\n- email-dst\n- target-email\n- whois-registrant-email\n-" + " whois-registrant-phone\n- text\n- whois-registrant-name\n- whois-registrar\n- whois-creation-date" + ), + "output": ( + "MISP attributes mapped from the result of the query on PassiveTotal, included in the following list:\n-" + " hostname\n- domain\n- ip-src\n- ip-dst\n- x509-fingerprint-sha1\n- email-src\n- email-dst\n- target-email\n-" + " whois-registrant-email\n- whois-registrant-phone\n- text\n- whois-registrant-name\n- whois-registrar\n-" + " whois-creation-date\n- md5\n- sha1\n- sha256\n- link" + ), } -moduleconfig = ['username', 'api_key'] +moduleconfig = ["username", "api_key"] query_playbook = [ - {'inputs': ['ip-src', 'ip-dst', 'hostname', 'domain'], - 'services': ['whois', 'ssl', 'dns', 'enrichment'], - 'name': 'generic'}, - {'inputs': ['whois-registrant-email', 'whois-registrant-phone', - 'whois-registrant-name', 'email-src', 'email-dst', - 'target-email'], - 'services': ['whois'], - 'name': 'reverse-whois'}, - {'inputs': ['x509-fingerprint-sha1'], - 'services': ['ssl'], - 'name': 'ssl-history'}, + { + "inputs": ["ip-src", "ip-dst", "hostname", "domain"], + "services": ["whois", "ssl", "dns", "enrichment"], + "name": "generic", + }, + { + "inputs": [ + "whois-registrant-email", + "whois-registrant-phone", + "whois-registrant-name", + "email-src", + "email-dst", + "target-email", + ], + "services": ["whois"], + "name": "reverse-whois", + }, + {"inputs": ["x509-fingerprint-sha1"], "services": ["ssl"], "name": "ssl-history"}, ] def query_finder(request): """Find the query value in the client request.""" - for item in mispattributes['input']: + for item in mispattributes["input"]: if not request.get(item, None): continue playbook = None for x in query_playbook: - if item not in x['inputs']: + if item not in x["inputs"]: continue playbook = x break - return {'type': item, 'value': request.get(item), 'playbook': playbook} + return {"type": item, "value": request.get(item), "playbook": playbook} def build_profile(request): """Check the incoming request for a valid configuration.""" - output = {'success': False} - config = request.get('config', None) + output = {"success": False} + config = request.get("config", None) if not config: - misperrors['error'] = "Configuration is missing from the request." + misperrors["error"] = "Configuration is missing from the request." return output for item in moduleconfig: if config.get(item, None): continue - misperrors['error'] = "PassiveTotal authentication is missing." + misperrors["error"] = "PassiveTotal authentication is missing." return output - profile = {'success': True, 'config': config} + profile = {"success": True, "config": config} profile.update(query_finder(request)) return profile @@ -102,27 +155,32 @@ def _generate_request_instance(conf, request_type): :param request_type: Type of client to load :return: Loaded PassiveTotal client """ - pt_username = conf.get('username') - pt_api_key = conf.get('api_key') - - class_lookup = {'dns': 'DnsRequest', 'whois': 'WhoisRequest', - 'ssl': 'SslRequest', 'enrichment': 'EnrichmentRequest', - 'attributes': 'AttributeRequest'} + pt_username = conf.get("username") + pt_api_key = conf.get("api_key") + + class_lookup = { + "dns": "DnsRequest", + "whois": "WhoisRequest", + "ssl": "SslRequest", + "enrichment": "EnrichmentRequest", + "attributes": "AttributeRequest", + } class_name = class_lookup[request_type] - mod = __import__('passivetotal.libs.%s' % request_type, - fromlist=[class_name]) + mod = __import__("passivetotal.libs.%s" % request_type, fromlist=[class_name]) loaded = getattr(mod, class_name) - headers = {'PT-INTEGRATION': 'MISP'} + headers = {"PT-INTEGRATION": "MISP"} authenticated = loaded(pt_username, pt_api_key, headers=headers) return authenticated def _has_error(results): """Check to see if there's an error in place and log it.""" - if 'error' in results: - msg = "%s - %s" % (results['error']['message'], - results['error']['developer_message']) - misperrors['error'] = msg + if "error" in results: + msg = "%s - %s" % ( + results["error"]["message"], + results["error"]["developer_message"], + ) + misperrors["error"] = msg return True return False @@ -136,10 +194,10 @@ def process_ssl_details(instance, query): err = _has_error(details) if err: raise Exception("We hit an error, time to bail!") - if details.get('message') and details['message'].startswith('quota_exceeded'): + if details.get("message") and details["message"].startswith("quota_exceeded"): raise Exception("API quota exceeded.") values = {value for value in details.values() if value} - txt = [{'types': ['ssl-cert-attributes'], 'values': list(values)}] + txt = [{"types": ["ssl-cert-attributes"], "values": list(values)}] log.debug("SSL Details: ending") return txt @@ -150,26 +208,26 @@ def process_ssl_history(instance, query): log.debug("SSL History: starting") type_map = { - 'ip': ['ip-src', 'ip-dst'], - 'domain': ['domain', 'hostname'], - 'sha1': ['x509-fingerprint-sha1'] + "ip": ["ip-src", "ip-dst"], + "domain": ["domain", "hostname"], + "sha1": ["x509-fingerprint-sha1"], } - hits = {'ip': list(), 'sha1': list(), 'domain': list()} + hits = {"ip": list(), "sha1": list(), "domain": list()} history = instance.get_ssl_certificate_history(query=query) err = _has_error(history) if err: raise Exception("We hit an error, time to bail!") - if history.get('message') and history['message'].startswith('quota_exceeded'): + if history.get("message") and history["message"].startswith("quota_exceeded"): raise Exception("API quota exceeded.") - for item in history.get('results', []): - hits['ip'] += item.get('ipAddresses', []) - hits['sha1'].append(item['sha1']) - hits['domain'] += item.get('domains', []) + for item in history.get("results", []): + hits["ip"] += item.get("ipAddresses", []) + hits["sha1"].append(item["sha1"]) + hits["domain"] += item.get("domains", []) tmp = list() for key, value in hits.items(): - txt = {'types': type_map[key], 'values': list(set(value))} + txt = {"types": type_map[key], "values": list(set(value))} tmp.append(txt) log.debug("SSL Details: ending") @@ -185,18 +243,23 @@ def process_whois_details(instance, query): err = _has_error(details) if err: raise Exception("We hit an error, time to bail!") - if details.get('message') and details['message'].startswith('quota_exceeded'): + if details.get("message") and details["message"].startswith("quota_exceeded"): raise Exception("API quota exceeded.") - if details.get('contactEmail', None): - tmp.append({'types': ['whois-registrant-email'], 'values': [details.get('contactEmail')]}) - phones = details['compact']['telephone']['raw'] - tmp.append({'types': ['whois-registrant-phone'], 'values': phones}) - names = details['compact']['name']['raw'] - tmp.append({'types': ['whois-registrant-name'], 'values': names}) - if details.get('registrar', None): - tmp.append({'types': ['whois-registrar'], 'values': [details.get('registrar')]}) - if details.get('registered', None): - tmp.append({'types': ['whois-creation-date'], 'values': [details.get('registered')]}) + if details.get("contactEmail", None): + tmp.append( + { + "types": ["whois-registrant-email"], + "values": [details.get("contactEmail")], + } + ) + phones = details["compact"]["telephone"]["raw"] + tmp.append({"types": ["whois-registrant-phone"], "values": phones}) + names = details["compact"]["name"]["raw"] + tmp.append({"types": ["whois-registrant-name"], "values": names}) + if details.get("registrar", None): + tmp.append({"types": ["whois-registrar"], "values": [details.get("registrar")]}) + if details.get("registered", None): + tmp.append({"types": ["whois-creation-date"], "values": [details.get("registered")]}) log.debug("WHOIS Details: ending") return tmp @@ -205,27 +268,27 @@ def process_whois_details(instance, query): def process_whois_search(instance, query, qtype): """Process a WHOIS search for a specific field value.""" log.debug("WHOIS Search: starting") - if qtype in ['whois-registrant-email', 'email-src', 'email-dst', 'target-email']: - field_type = 'email' - if qtype in ['whois-registrant-phone']: - field_type = 'phone' - if qtype in ['whois-registrant-name']: - field_type = 'name' + if qtype in ["whois-registrant-email", "email-src", "email-dst", "target-email"]: + field_type = "email" + if qtype in ["whois-registrant-phone"]: + field_type = "phone" + if qtype in ["whois-registrant-name"]: + field_type = "name" domains = list() search = instance.search_whois_by_field(field=field_type, query=query) err = _has_error(search) if err: raise Exception("We hit an error, time to bail!") - if search.get('message') and search['message'].startswith('quota_exceeded'): + if search.get("message") and search["message"].startswith("quota_exceeded"): raise Exception("API quota exceeded.") - for item in search.get('results', []): - domain = item.get('domain', None) + for item in search.get("results", []): + domain = item.get("domain", None) if not domain: continue domains.append(domain) - tmp = [{'types': ['hostname', 'domain'], 'values': list(set(domains))}] + tmp = [{"types": ["hostname", "domain"], "values": list(set(domains))}] log.debug("WHOIS Search: ending") return tmp @@ -239,12 +302,12 @@ def process_passive_dns(instance, query): err = _has_error(pdns) if err: raise Exception("We hit an error, time to bail!") - if pdns.get('message') and pdns['message'].startswith('quota_exceeded'): + if pdns.get("message") and pdns["message"].startswith("quota_exceeded"): raise Exception("API quota exceeded.") if is_ip(query): - tmp = [{'types': ['domain', 'hostname'], 'values': pdns.get('results', [])}] + tmp = [{"types": ["domain", "hostname"], "values": pdns.get("results", [])}] else: - tmp = [{'types': ['ip-src', 'ip-dst'], 'values': pdns.get('results', [])}] + tmp = [{"types": ["ip-src", "ip-dst"], "values": pdns.get("results", [])}] log.debug("Passive DNS: ending") return tmp @@ -258,12 +321,12 @@ def process_osint(instance, query): err = _has_error(osint) if err: raise Exception("We hit an error, time to bail!") - if osint.get('message') and osint['message'].startswith('quota_exceeded'): + if osint.get("message") and osint["message"].startswith("quota_exceeded"): raise Exception("API quota exceeded.") - for item in osint.get('results', []): - urls.append(item['sourceUrl']) + for item in osint.get("results", []): + urls.append(item["sourceUrl"]) - tmp = [{'types': ['link'], 'values': urls}] + tmp = [{"types": ["link"], "values": urls}] log.debug("OSINT: ending") return tmp @@ -272,29 +335,29 @@ def process_osint(instance, query): def process_malware(instance, query): """Process malware samples.""" log.debug("Malware: starting") - content = {'hashes': list(), 'urls': list()} + content = {"hashes": list(), "urls": list()} malware = instance.get_malware(query=query) err = _has_error(malware) if err: raise Exception("We hit an error, time to bail!") - if malware.get('message') and malware['message'].startswith('quota_exceeded'): + if malware.get("message") and malware["message"].startswith("quota_exceeded"): raise Exception("API quota exceeded.") - for item in malware.get('results', []): - content['hashes'].append(item['sample']) - content['urls'].append(item['sourceUrl']) + for item in malware.get("results", []): + content["hashes"].append(item["sample"]) + content["urls"].append(item["sourceUrl"]) - tmp = [{'types': ['link'], 'values': content['urls']}] - hashes = {'md5': list(), 'sha1': list(), 'sha256': list()} - for h in content['hashes']: + tmp = [{"types": ["link"], "values": content["urls"]}] + hashes = {"md5": list(), "sha1": list(), "sha256": list()} + for h in content["hashes"]: if len(h) == 32: - hashes['md5'].append(h) + hashes["md5"].append(h) elif len(h) == 41: - hashes['sha1'].append(h) + hashes["sha1"].append(h) elif len(h) == 64: - hashes['sha256'].append(h) - tmp += [{'types': ['md5'], 'values': hashes['md5']}] - tmp += [{'types': ['sha1'], 'values': hashes['sha1']}] - tmp += [{'types': ['sha256'], 'values': hashes['sha256']}] + hashes["sha256"].append(h) + tmp += [{"types": ["md5"], "values": hashes["md5"]}] + tmp += [{"types": ["sha1"], "values": hashes["sha1"]}] + tmp += [{"types": ["sha256"], "values": hashes["sha256"]}] log.debug("Malware: ending") return tmp @@ -306,44 +369,43 @@ def handler(q=False): request = json.loads(q) profile = build_profile(request) - if not profile['success']: - log.error(misperrors['error']) + if not profile["success"]: + log.error(misperrors["error"]) return misperrors - output = {'results': list()} + output = {"results": list()} instances = dict() - for service in profile['playbook']['services']: - instances[service] = _generate_request_instance( - profile['config'], service) + for service in profile["playbook"]["services"]: + instances[service] = _generate_request_instance(profile["config"], service) - play_type = profile['playbook']['name'] - query = profile['value'] - qtype = profile['type'] + play_type = profile["playbook"]["name"] + query = profile["value"] + qtype = profile["type"] try: - if play_type == 'generic': - results = process_passive_dns(instances['dns'], query) - output['results'] += results - results = process_whois_details(instances['whois'], query) - output['results'] += results - results = process_ssl_history(instances['ssl'], query) - output['results'] += results - results = process_osint(instances['enrichment'], query) - output['results'] += results - results = process_malware(instances['enrichment'], query) - output['results'] += results - elif play_type == 'reverse-whois': - results = process_whois_search(instances['whois'], query, qtype) - output['results'] += results - elif play_type == 'ssl-history': - results = process_ssl_details(instances['ssl'], query) - output['results'] += results - results = process_ssl_history(instances['ssl'], query) - output['results'] += results + if play_type == "generic": + results = process_passive_dns(instances["dns"], query) + output["results"] += results + results = process_whois_details(instances["whois"], query) + output["results"] += results + results = process_ssl_history(instances["ssl"], query) + output["results"] += results + results = process_osint(instances["enrichment"], query) + output["results"] += results + results = process_malware(instances["enrichment"], query) + output["results"] += results + elif play_type == "reverse-whois": + results = process_whois_search(instances["whois"], query, qtype) + output["results"] += results + elif play_type == "ssl-history": + results = process_ssl_details(instances["ssl"], query) + output["results"] += results + results = process_ssl_history(instances["ssl"], query) + output["results"] += results else: log.error("Unsupported query pattern issued.") except Exception as e: - misperrors['error'] = e.__str__() + misperrors["error"] = e.__str__() return misperrors return output @@ -354,5 +416,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/pdf_enrich.py b/misp_modules/modules/expansion/pdf_enrich.py index 15231c095..2b3622302 100644 --- a/misp_modules/modules/expansion/pdf_enrich.py +++ b/misp_modules/modules/expansion/pdf_enrich.py @@ -1,24 +1,27 @@ -import json import binascii +import io +import json + import np import pdftotext -import io -misperrors = {'error': 'Error'} -mispattributes = {'input': ['attachment'], - 'output': ['freetext', 'text']} +misperrors = {"error": "Error"} +mispattributes = {"input": ["attachment"], "output": ["freetext", "text"]} moduleinfo = { - 'version': '0.1', - 'author': 'Sascha Rommelfangen', - 'description': 'Module to extract freetext from a PDF document.', - 'module-type': ['expansion'], - 'name': 'PDF Enrich', - 'logo': 'pdf.jpg', - 'requirements': ['pdftotext: Python library to extract text from PDF.'], - 'features': 'The module reads the text contained in a PDF document. The result is passed to the freetext import parser so IoCs can be extracted out of it.', - 'references': [], - 'input': 'Attachment attribute containing a PDF document.', - 'output': 'Text and freetext parsed from the document.', + "version": "0.1", + "author": "Sascha Rommelfangen", + "description": "Module to extract freetext from a PDF document.", + "module-type": ["expansion"], + "name": "PDF Enrich", + "logo": "pdf.jpg", + "requirements": ["pdftotext: Python library to extract text from PDF."], + "features": ( + "The module reads the text contained in a PDF document. The result is passed to the freetext import parser so" + " IoCs can be extracted out of it." + ), + "references": [], + "input": "Attachment attribute containing a PDF document.", + "output": "Text and freetext parsed from the document.", } moduleconfig = [] @@ -28,24 +31,32 @@ def handler(q=False): if q is False: return False q = json.loads(q) - filename = q['attachment'] + filename = q["attachment"] try: - pdf_array = np.frombuffer(binascii.a2b_base64(q['data']), np.uint8) + pdf_array = np.frombuffer(binascii.a2b_base64(q["data"]), np.uint8) except Exception as e: print(e) err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?" - misperrors['error'] = err + misperrors["error"] = err print(err) return misperrors pdf_file = io.BytesIO(pdf_array) try: pdf_content = "\n\n".join(pdftotext.PDF(pdf_file)) - return {'results': [{'types': ['freetext'], 'values': pdf_content, 'comment': "PDF-to-text from file " + filename}]} + return { + "results": [ + { + "types": ["freetext"], + "values": pdf_content, + "comment": "PDF-to-text from file " + filename, + } + ] + } except Exception as e: print(e) err = "Couldn't analyze file as PDF. Error was: " + str(e) - misperrors['error'] = err + misperrors["error"] = err return misperrors @@ -54,5 +65,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/pptx_enrich.py b/misp_modules/modules/expansion/pptx_enrich.py index 4a3b2b5f6..829dc6606 100644 --- a/misp_modules/modules/expansion/pptx_enrich.py +++ b/misp_modules/modules/expansion/pptx_enrich.py @@ -1,24 +1,27 @@ -import json import binascii +import io +import json + import np from pptx import Presentation -import io -misperrors = {'error': 'Error'} -mispattributes = {'input': ['attachment'], - 'output': ['freetext', 'text']} +misperrors = {"error": "Error"} +mispattributes = {"input": ["attachment"], "output": ["freetext", "text"]} moduleinfo = { - 'version': '0.1', - 'author': 'Sascha Rommelfangen', - 'description': 'Module to extract freetext from a .pptx document.', - 'module-type': ['expansion'], - 'name': 'PPTX Enrich', - 'logo': 'pptx.png', - 'requirements': ['pptx: Python library to read PowerPoint files.'], - 'features': 'The module reads the text contained in a .pptx document. The result is passed to the freetext import parser so IoCs can be extracted out of it.', - 'references': [], - 'input': 'Attachment attribute containing a .pptx document.', - 'output': 'Text and freetext parsed from the document.', + "version": "0.1", + "author": "Sascha Rommelfangen", + "description": "Module to extract freetext from a .pptx document.", + "module-type": ["expansion"], + "name": "PPTX Enrich", + "logo": "pptx.png", + "requirements": ["pptx: Python library to read PowerPoint files."], + "features": ( + "The module reads the text contained in a .pptx document. The result is passed to the freetext import parser so" + " IoCs can be extracted out of it." + ), + "references": [], + "input": "Attachment attribute containing a .pptx document.", + "output": "Text and freetext parsed from the document.", } moduleconfig = [] @@ -28,13 +31,13 @@ def handler(q=False): if q is False: return False q = json.loads(q) - filename = q['attachment'] + filename = q["attachment"] try: - pptx_array = np.frombuffer(binascii.a2b_base64(q['data']), np.uint8) + pptx_array = np.frombuffer(binascii.a2b_base64(q["data"]), np.uint8) except Exception as e: print(e) err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?" - misperrors['error'] = err + misperrors["error"] = err print(err) return misperrors @@ -47,12 +50,24 @@ def handler(q=False): if hasattr(shape, "text"): print(shape.text) ppt_content = ppt_content + "\n" + shape.text - return {'results': [{'types': ['freetext'], 'values': ppt_content, 'comment': ".pptx-to-text from file " + filename}, - {'types': ['text'], 'values': ppt_content, 'comment': ".pptx-to-text from file " + filename}]} + return { + "results": [ + { + "types": ["freetext"], + "values": ppt_content, + "comment": ".pptx-to-text from file " + filename, + }, + { + "types": ["text"], + "values": ppt_content, + "comment": ".pptx-to-text from file " + filename, + }, + ] + } except Exception as e: print(e) err = "Couldn't analyze file as .pptx. Error was: " + str(e) - misperrors['error'] = err + misperrors["error"] = err return misperrors @@ -61,5 +76,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/qintel_qsentry.py b/misp_modules/modules/expansion/qintel_qsentry.py index 609ed01f1..8a0e6b172 100644 --- a/misp_modules/modules/expansion/qintel_qsentry.py +++ b/misp_modules/modules/expansion/qintel_qsentry.py @@ -1,67 +1,66 @@ -import logging import json +import logging -from pymisp import MISPAttribute, MISPEvent, MISPTag, MISPObject -from . import check_input_attribute, checking_error, standard_error_message - +from pymisp import MISPAttribute, MISPEvent, MISPObject, MISPTag from qintel_helper import search_qsentry -logger = logging.getLogger('qintel_qsentry') +from . import check_input_attribute, checking_error, standard_error_message + +logger = logging.getLogger("qintel_qsentry") logger.setLevel(logging.DEBUG) moduleinfo = { - 'version': '1.0', - 'author': 'Qintel, LLC', - 'description': 'A hover and expansion module which queries Qintel QSentry for ip reputation data', - 'module-type': ['hover', 'expansion'], - 'name': 'Qintel QSentry Lookup', - 'logo': 'qintel.png', - 'requirements': ['A Qintel API token'], - 'features': 'This module takes an ip-address (ip-src or ip-dst) attribute as input, and queries the Qintel QSentry API to retrieve ip reputation data', - 'references': ['https://www.qintel.com/products/qsentry/'], - 'input': 'ip address attribute', - 'output': '', - 'ouput': 'Objects containing the enriched IP, threat tags, last seen attributes and associated Autonomous System information', + "version": "1.0", + "author": "Qintel, LLC", + "description": "A hover and expansion module which queries Qintel QSentry for ip reputation data", + "module-type": ["hover", "expansion"], + "name": "Qintel QSentry Lookup", + "logo": "qintel.png", + "requirements": ["A Qintel API token"], + "features": ( + "This module takes an ip-address (ip-src or ip-dst) attribute as input, and queries the Qintel QSentry API to" + " retrieve ip reputation data" + ), + "references": ["https://www.qintel.com/products/qsentry/"], + "input": "ip address attribute", + "output": "", + "ouput": ( + "Objects containing the enriched IP, threat tags, last seen attributes and associated Autonomous System" + " information" + ), } -moduleconfig = ['token', 'remote'] +moduleconfig = ["token", "remote"] -misperrors = {'error': 'Error'} +misperrors = {"error": "Error"} mispattributes = { - 'input': ['ip-src', 'ip-dst'], - 'output': ['ip-src', 'ip-dst', 'AS', 'freetext'], - 'format': 'misp_standard' + "input": ["ip-src", "ip-dst"], + "output": ["ip-src", "ip-dst", "AS", "freetext"], + "format": "misp_standard", } -TAG_COLOR = { - 'benign': '#27ae60', - 'suspicious': '#e6a902', - 'malicious': '#c0392b' -} +TAG_COLOR = {"benign": "#27ae60", "suspicious": "#e6a902", "malicious": "#c0392b"} CLIENT_HEADERS = { - 'User-Agent': f"MISP/{moduleinfo['version']}", + "User-Agent": f"MISP/{moduleinfo['version']}", } def _return_error(message): - misperrors['error'] = message + misperrors["error"] = message return misperrors def _make_tags(enriched_attr, result): - for tag in result['tags']: - color = TAG_COLOR['suspicious'] - if tag == 'criminal': - color = TAG_COLOR['malicious'] + for tag in result["tags"]: + color = TAG_COLOR["suspicious"] + if tag == "criminal": + color = TAG_COLOR["malicious"] t = MISPTag() - t.from_dict(**{ - 'name': f'qintel:tag="{tag}"', - 'colour': color - }) + t.from_dict(**{"name": f'qintel:tag="{tag}"', "colour": color}) enriched_attr.add_tag(**t) return enriched_attr @@ -69,37 +68,43 @@ def _make_tags(enriched_attr, result): def _make_enriched_attr(event, result, orig_attr): - enriched_object = MISPObject('Qintel Threat Enrichment') - enriched_object.add_reference(orig_attr.uuid, 'related-to') + enriched_object = MISPObject("Qintel Threat Enrichment") + enriched_object.add_reference(orig_attr.uuid, "related-to") enriched_attr = MISPAttribute() - enriched_attr.from_dict(**{ - 'value': orig_attr.value, - 'type': orig_attr.type, - 'distribution': 0, - 'object_relation': 'enriched-attr', - 'to_ids': orig_attr.to_ids - }) + enriched_attr.from_dict( + **{ + "value": orig_attr.value, + "type": orig_attr.type, + "distribution": 0, + "object_relation": "enriched-attr", + "to_ids": orig_attr.to_ids, + } + ) enriched_attr = _make_tags(enriched_attr, result) enriched_object.add_attribute(**enriched_attr) comment_attr = MISPAttribute() - comment_attr.from_dict(**{ - 'value': '\n'.join(result.get('descriptions', [])), - 'type': 'text', - 'object_relation': 'descriptions', - 'distribution': 0 - }) + comment_attr.from_dict( + **{ + "value": "\n".join(result.get("descriptions", [])), + "type": "text", + "object_relation": "descriptions", + "distribution": 0, + } + ) enriched_object.add_attribute(**comment_attr) last_seen = MISPAttribute() - last_seen.from_dict(**{ - 'value': result.get('last_seen'), - 'type': 'datetime', - 'object_relation': 'last-seen', - 'distribution': 0 - }) + last_seen.from_dict( + **{ + "value": result.get("last_seen"), + "type": "datetime", + "object_relation": "last-seen", + "distribution": 0, + } + ) enriched_object.add_attribute(**last_seen) event.add_attribute(**orig_attr) @@ -110,25 +115,29 @@ def _make_enriched_attr(event, result, orig_attr): def _make_asn_attr(event, result, orig_attr): - asn_object = MISPObject('asn') - asn_object.add_reference(orig_attr.uuid, 'related-to') + asn_object = MISPObject("asn") + asn_object.add_reference(orig_attr.uuid, "related-to") asn_attr = MISPAttribute() - asn_attr.from_dict(**{ - 'type': 'AS', - 'value': result.get('asn'), - 'object_relation': 'asn', - 'distribution': 0 - }) + asn_attr.from_dict( + **{ + "type": "AS", + "value": result.get("asn"), + "object_relation": "asn", + "distribution": 0, + } + ) asn_object.add_attribute(**asn_attr) org_attr = MISPAttribute() - org_attr.from_dict(**{ - 'type': 'text', - 'value': result.get('asn_name', 'unknown').title(), - 'object_relation': 'description', - 'distribution': 0 - }) + org_attr.from_dict( + **{ + "type": "text", + "value": result.get("asn_name", "unknown").title(), + "object_relation": "description", + "distribution": 0, + } + ) asn_object.add_attribute(**org_attr) event.add_object(**asn_object) @@ -138,10 +147,10 @@ def _make_asn_attr(event, result, orig_attr): def _format_hover(event, result): - enriched_object = event.get_objects_by_name('Qintel Threat Enrichment')[0] + enriched_object = event.get_objects_by_name("Qintel Threat Enrichment")[0] - tags = ', '.join(result.get('tags')) - enriched_object.add_attribute('Tags', type='text', value=tags) + tags = ", ".join(result.get("tags")) + enriched_object.add_attribute("Tags", type="text", value=tags) return event @@ -166,23 +175,22 @@ def _check_config(config): if not isinstance(config, dict): return False - if config.get('token', '') == '': + if config.get("token", "") == "": return False return True def _check_request(request): - if not request.get('attribute'): - return f'{standard_error_message}, {checking_error}' + if not request.get("attribute"): + return f"{standard_error_message}, {checking_error}" - check_reqs = ('type', 'value') - if not check_input_attribute(request['attribute'], - requirements=check_reqs): - return f'{standard_error_message}, {checking_error}' + check_reqs = ("type", "value") + if not check_input_attribute(request["attribute"], requirements=check_reqs): + return f"{standard_error_message}, {checking_error}" - if request['attribute']['type'] not in mispattributes['input']: - return 'Unsupported attribute type' + if request["attribute"]["type"] not in mispattributes["input"]: + return "Unsupported attribute type" def handler(q=False): @@ -190,34 +198,30 @@ def handler(q=False): return False request = json.loads(q) - config = request.get('config') + config = request.get("config") if not _check_config(config): - return _return_error('Missing Qintel token') + return _return_error("Missing Qintel token") check_request_error = _check_request(request) if check_request_error: return _return_error(check_request_error) - search_args = { - 'token': config['token'], - 'remote': config.get('remote') - } + search_args = {"token": config["token"], "remote": config.get("remote")} try: - result = search_qsentry(request['attribute']['value'], **search_args) + result = search_qsentry(request["attribute"]["value"], **search_args) except Exception as e: return _return_error(str(e)) - event = _format_result(request['attribute'], result) - if not request.get('event_id'): + event = _format_result(request["attribute"], result) + if not request.get("event_id"): event = _format_hover(event, result) event = json.loads(event.to_json()) - ret_result = {key: event[key] for key in ('Attribute', 'Object') if key - in event} - return {'results': ret_result} + ret_result = {key: event[key] for key in ("Attribute", "Object") if key in event} + return {"results": ret_result} def introspection(): @@ -225,5 +229,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/qrcode.py b/misp_modules/modules/expansion/qrcode.py index a44d311fc..dd0577ccf 100644 --- a/misp_modules/modules/expansion/qrcode.py +++ b/misp_modules/modules/expansion/qrcode.py @@ -1,33 +1,38 @@ +import binascii import json -from pyzbar import pyzbar -import cv2 import re -import binascii + +import cv2 import np +from pyzbar import pyzbar -misperrors = {'error': 'Error'} -mispattributes = {'input': ['attachment'], - 'output': ['url', 'btc']} +misperrors = {"error": "Error"} +mispattributes = {"input": ["attachment"], "output": ["url", "btc"]} moduleinfo = { - 'version': '0.1', - 'author': 'Sascha Rommelfangen', - 'description': 'Module to decode QR codes.', - 'module-type': ['expansion', 'hover'], - 'name': 'QR Code Decode', - 'logo': '', - 'requirements': ['cv2: The OpenCV python library.', 'pyzbar: Python library to read QR codes.'], - 'features': 'The module reads the QR code and returns the related address, which can be an URL or a bitcoin address.', - 'references': [], - 'input': 'A QR code stored as attachment attribute.', - 'output': 'The URL or bitcoin address the QR code is pointing to.', + "version": "0.1", + "author": "Sascha Rommelfangen", + "description": "Module to decode QR codes.", + "module-type": ["expansion", "hover"], + "name": "QR Code Decode", + "logo": "", + "requirements": [ + "cv2: The OpenCV python library.", + "pyzbar: Python library to read QR codes.", + ], + "features": ( + "The module reads the QR code and returns the related address, which can be an URL or a bitcoin address." + ), + "references": [], + "input": "A QR code stored as attachment attribute.", + "output": "The URL or bitcoin address the QR code is pointing to.", } debug = True debug_prefix = "[DEBUG] QR Code module: " # format example: bitcoin:1GXZ6v7FZzYBEnoRaG77SJxhu7QkvQmFuh?amount=0.15424 # format example: http://example.com -cryptocurrencies = ['bitcoin'] -schemas = ['http://', 'https://', 'ftp://'] +cryptocurrencies = ["bitcoin"] +schemas = ["http://", "https://", "ftp://"] moduleconfig = [] @@ -35,12 +40,12 @@ def handler(q=False): if q is False: return False q = json.loads(q) - filename = q['attachment'] + filename = q["attachment"] try: - img_array = np.frombuffer(binascii.a2b_base64(q['data']), np.uint8) + img_array = np.frombuffer(binascii.a2b_base64(q["data"]), np.uint8) except Exception as e: err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?" - misperrors['error'] = err + misperrors["error"] = err print(err) print(e) return misperrors @@ -58,16 +63,24 @@ def handler(q=False): for item in cryptocurrencies: if item in result: try: - currency, address, extra = re.split(r'\:|\?', result) + currency, address, extra = re.split(r"\:|\?", result) except Exception as e: print(e) if currency in cryptocurrencies: try: - amount = re.split('=', extra)[1] + amount = re.split("=", extra)[1] if debug: print(debug_prefix + address) print(debug_prefix + amount) - return {'results': [{'types': ['btc'], 'values': address, 'comment': "BTC: " + amount + " from file " + filename}]} + return { + "results": [ + { + "types": ["btc"], + "values": address, + "comment": "BTC: " + amount + " from file " + filename, + } + ] + } except Exception as e: print(e) else: @@ -78,15 +91,31 @@ def handler(q=False): url = result if debug: print(debug_prefix + url) - return {'results': [{'types': ['url'], 'values': url, 'comment': "from QR code of file " + filename}]} + return { + "results": [ + { + "types": ["url"], + "values": url, + "comment": "from QR code of file " + filename, + } + ] + } except Exception as e: print(e) else: try: - return {'results': [{'types': ['text'], 'values': result, 'comment': "from QR code of file " + filename}]} + return { + "results": [ + { + "types": ["text"], + "values": result, + "comment": "from QR code of file " + filename, + } + ] + } except Exception as e: print(e) - misperrors['error'] = "Couldn't decode QR code in attachment." + misperrors["error"] = "Couldn't decode QR code in attachment." return misperrors @@ -95,5 +124,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/ransomcoindb.py b/misp_modules/modules/expansion/ransomcoindb.py index 20b5ebfdb..273c9de89 100644 --- a/misp_modules/modules/expansion/ransomcoindb.py +++ b/misp_modules/modules/expansion/ransomcoindb.py @@ -1,7 +1,9 @@ import json + +from pymisp import MISPObject + from . import check_input_attribute, checking_error, standard_error_message from ._ransomcoindb import ransomcoindb -from pymisp import MISPObject copyright = """ Copyright 2019 (C) by Aaron Kaplan , all rights reserved. @@ -11,28 +13,40 @@ debug = False -misperrors = {'error': 'Error'} +misperrors = {"error": "Error"} # mispattributes = {'input': ['sha1', 'sha256', 'md5', 'btc', 'xmr', 'dash' ], 'output': ['btc', 'sha1', 'sha256', 'md5', 'freetext']} -mispattributes = {'input': ['sha1', 'sha256', 'md5', 'btc'], 'output': ['btc', 'sha1', 'sha256', 'md5', 'freetext'], 'format': 'misp_standard'} +mispattributes = { + "input": ["sha1", "sha256", "md5", "btc"], + "output": ["btc", "sha1", "sha256", "md5", "freetext"], + "format": "misp_standard", +} moduleinfo = { - 'version': '0.1', - 'author': 'Aaron Kaplan', - 'description': 'Module to access the ransomcoinDB (see https://ransomcoindb.concinnity-risks.com)', - 'module-type': ['expansion', 'hover'], - 'name': 'RandomcoinDB Lookup', - 'logo': '', - 'requirements': ['A ransomcoinDB API key.'], - 'features': 'The module takes either a hash attribute or a btc attribute as input to query the ransomcoinDB API for some additional data.\n\nIf the input is a btc address, we will get the associated hashes returned in a file MISP object. If we query ransomcoinDB with a hash, the response contains the associated btc addresses returned as single MISP btc attributes.', - 'references': ['https://ransomcoindb.concinnity-risks.com'], - 'input': 'A hash (md5, sha1 or sha256) or btc attribute.', - 'output': 'Hashes associated to a btc address or btc addresses associated to a hash.', - 'descrption': 'Module to access the ransomcoinDB with a hash or btc address attribute and get the associated btc address of hashes.', + "version": "0.1", + "author": "Aaron Kaplan", + "description": "Module to access the ransomcoinDB (see https://ransomcoindb.concinnity-risks.com)", + "module-type": ["expansion", "hover"], + "name": "RandomcoinDB Lookup", + "logo": "", + "requirements": ["A ransomcoinDB API key."], + "features": ( + "The module takes either a hash attribute or a btc attribute as input to query the ransomcoinDB API for some" + " additional data.\n\nIf the input is a btc address, we will get the associated hashes returned in a file MISP" + " object. If we query ransomcoinDB with a hash, the response contains the associated btc addresses returned as" + " single MISP btc attributes." + ), + "references": ["https://ransomcoindb.concinnity-risks.com"], + "input": "A hash (md5, sha1 or sha256) or btc attribute.", + "output": "Hashes associated to a btc address or btc addresses associated to a hash.", + "descrption": ( + "Module to access the ransomcoinDB with a hash or btc address attribute and get the associated btc address of" + " hashes." + ), } -moduleconfig = ['api-key'] +moduleconfig = ["api-key"] def handler(q=False): - """ the main handler function which gets a JSON dict as input and returns a results dict """ + """the main handler function which gets a JSON dict as input and returns a results dict""" if q is False: return False @@ -40,10 +54,10 @@ def handler(q=False): q = json.loads(q) if "config" not in q or "api-key" not in q["config"]: return {"error": "Ransomcoindb API key is missing"} - if not q.get('attribute') or not check_input_attribute(q['attribute'], requirements=('type', 'value')): - return {'error': f'{standard_error_message}, {checking_error}.'} - if q['attribute']['type'] not in mispattributes['input']: - return {'error': 'Unsupported attribute type.'} + if not q.get("attribute") or not check_input_attribute(q["attribute"], requirements=("type", "value")): + return {"error": f"{standard_error_message}, {checking_error}."} + if q["attribute"]["type"] not in mispattributes["input"]: + return {"error": "Unsupported attribute type."} api_key = q["config"]["api-key"] r = {"results": []} @@ -53,23 +67,23 @@ def handler(q=False): 'module': 'ransomcoindb', 'persistent': 1} """ - attribute = q['attribute'] - answer = ransomcoindb.get_data_by('BTC', attribute['type'], attribute['value'], api_key) + attribute = q["attribute"] + answer = ransomcoindb.get_data_by("BTC", attribute["type"], attribute["value"], api_key) """ The results data type should be: r = { 'results': [ {'types': 'md5', 'values': [ a list of all md5s or all binaries related to this btc address ] } ] } """ - if attribute['type'] in ['md5', 'sha1', 'sha256']: - r['results'].append({'types': 'btc', 'values': [a['btc'] for a in answer]}) - elif attribute['type'] == 'btc': + if attribute["type"] in ["md5", "sha1", "sha256"]: + r["results"].append({"types": "btc", "values": [a["btc"] for a in answer]}) + elif attribute["type"] == "btc": # better: create a MISP object files = [] for a in answer: - obj = MISPObject('file') - obj.add_attribute('md5', a['md5']) - obj.add_attribute('sha1', a['sha1']) - obj.add_attribute('sha256', a['sha256']) + obj = MISPObject("file") + obj.add_attribute("md5", a["md5"]) + obj.add_attribute("sha1", a["sha1"]) + obj.add_attribute("sha256", a["sha256"]) files.append(obj) - r['results'] = {'Object': [json.loads(f.to_json()) for f in files]} + r["results"] = {"Object": [json.loads(f.to_json()) for f in files]} return r @@ -78,5 +92,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/rbl.py b/misp_modules/modules/expansion/rbl.py index 408ca5188..b7446e399 100644 --- a/misp_modules/modules/expansion/rbl.py +++ b/misp_modules/modules/expansion/rbl.py @@ -1,28 +1,30 @@ import json -import sys -try: - import dns.resolver -except ImportError: - print("dnspython3 is missing, use 'pip install dnspython3' to install it.") - sys.exit(0) +import dns.resolver -misperrors = {'error': 'Error'} -mispattributes = {'input': ['ip-src', 'ip-dst'], 'output': ['text']} +misperrors = {"error": "Error"} +mispattributes = {"input": ["ip-src", "ip-dst"], "output": ["text"]} moduleinfo = { - 'version': '0.2', - 'author': 'Christian Studer', - 'description': 'Module to check an IPv4 address against known RBLs.', - 'module-type': ['expansion', 'hover'], - 'name': 'Real-time Blackhost Lists Lookup', - 'logo': '', - 'requirements': ['dnspython3: DNS python3 library'], - 'features': 'This module takes an IP address attribute as input and queries multiple know Real-time Blackhost Lists to check if they have already seen this IP address.\n\nWe display then all the information we get from those different sources.', - 'references': ['[RBLs list](https://github.com/MISP/misp-modules/blob/8817de476572a10a9c9d03258ec81ca70f3d926d/misp_modules/modules/expansion/rbl.py#L20)'], - 'input': 'IP address attribute.', - 'output': 'Text with additional data from Real-time Blackhost Lists about the IP address.', + "version": "0.2", + "author": "Christian Studer", + "description": "Module to check an IPv4 address against known RBLs.", + "module-type": ["expansion", "hover"], + "name": "Real-time Blackhost Lists Lookup", + "logo": "", + "requirements": ["dnspython3: DNS python3 library"], + "features": ( + "This module takes an IP address attribute as input and queries multiple know Real-time Blackhost Lists to" + " check if they have already seen this IP address.\n\nWe display then all the information we get from those" + " different sources." + ), + "references": [ + "[RBLs" + " list](https://github.com/MISP/misp-modules/blob/8817de476572a10a9c9d03258ec81ca70f3d926d/misp_modules/modules/expansion/rbl.py#L20)" + ], + "input": "IP address attribute.", + "output": "Text with additional data from Real-time Blackhost Lists about the IP address.", } -moduleconfig = ['timeout'] +moduleconfig = ["timeout"] rbls = ( "spam.spamrats.com", @@ -80,7 +82,7 @@ "dnsbl.njabl.org", "relays.mail-abuse.org", "rbl.spamlab.com", - "all.bl.blocklist.de" + "all.bl.blocklist.de", ) @@ -88,33 +90,33 @@ def handler(q=False): if q is False: return False request = json.loads(q) - if request.get('ip-src'): - ip = request['ip-src'] - elif request.get('ip-dst'): - ip = request['ip-dst'] + if request.get("ip-src"): + ip = request["ip-src"] + elif request.get("ip-dst"): + ip = request["ip-dst"] else: - misperrors['error'] = "Unsupported attributes type" + misperrors["error"] = "Unsupported attributes type" return misperrors resolver = dns.resolver.Resolver() try: - timeout = float(request['config']['timeout']) + timeout = float(request["config"]["timeout"]) except (KeyError, ValueError): timeout = 0.4 resolver.timeout = timeout resolver.lifetime = timeout infos = {} - ipRev = '.'.join(ip.split('.')[::-1]) + ipRev = ".".join(ip.split(".")[::-1]) for rbl in rbls: - query = '{}.{}'.format(ipRev, rbl) + query = "{}.{}".format(ipRev, rbl) try: - txt = resolver.query(query, 'TXT') + txt = resolver.query(query, "TXT") infos[query] = [str(t) for t in txt] except Exception: continue result = "\n".join([f"{rbl}: {' - '.join(info)}" for rbl, info in infos.items()]) if not result: - return {'error': 'No data found by querying known RBLs'} - return {'results': [{'types': mispattributes.get('output'), 'values': result}]} + return {"error": "No data found by querying known RBLs"} + return {"results": [{"types": mispattributes.get("output"), "values": result}]} def introspection(): @@ -122,5 +124,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/recordedfuture.py b/misp_modules/modules/expansion/recordedfuture.py index ad6e4c6b4..2eb76d148 100644 --- a/misp_modules/modules/expansion/recordedfuture.py +++ b/misp_modules/modules/expansion/recordedfuture.py @@ -1,32 +1,40 @@ import json import logging -import requests -from requests.exceptions import ( - HTTPError, - ProxyError, - InvalidURL, - ConnectTimeout, - ConnectionError, -) -from typing import Optional, List, Tuple, Dict -from . import check_input_attribute, checking_error, standard_error_message -import platform import os +import platform +from typing import Dict, List, Optional, Tuple from urllib.parse import quote, urlparse -from pymisp import MISPAttribute, MISPEvent, MISPTag, MISPObject + +import requests +from pymisp import MISPAttribute, MISPEvent, MISPObject, MISPTag +from requests.exceptions import ConnectionError, ConnectTimeout, HTTPError, InvalidURL, ProxyError + +from . import check_input_attribute, checking_error, standard_error_message moduleinfo = { - 'version': '2.0.0', - 'author': 'Recorded Future', - 'description': 'Module to enrich attributes with threat intelligence from Recorded Future.', - 'module-type': ['expansion', 'hover'], - 'name': 'Recorded Future Enrich', - 'logo': 'recordedfuture.png', - 'requirements': ['A Recorded Future API token.'], - 'features': "Enrich an attribute to add a custom enrichment object to the event. The object contains a copy of the enriched attribute with added tags presenting risk score and triggered risk rules from Recorded Future. Malware and Threat Actors related to the enriched indicator in Recorded Future is matched against MISP's galaxy clusters and applied as galaxy tags. The custom enrichment object also includes a list of related indicators from Recorded Future (IP's, domains, hashes, URL's and vulnerabilities) added as additional attributes.", - 'references': ['https://www.recordedfuture.com/'], - 'input': 'A MISP attribute of one of the following types: ip, ip-src, ip-dst, domain, hostname, md5, sha1, sha256, uri, url, vulnerability, weakness.', - 'output': 'A MISP object containing a copy of the enriched attribute with added tags from Recorded Future and a list of new attributes related to the enriched attribute.', + "version": "2.0.0", + "author": "Recorded Future", + "description": "Module to enrich attributes with threat intelligence from Recorded Future.", + "module-type": ["expansion", "hover"], + "name": "Recorded Future Enrich", + "logo": "recordedfuture.png", + "requirements": ["A Recorded Future API token."], + "features": ( + "Enrich an attribute to add a custom enrichment object to the event. The object contains a copy of the enriched" + " attribute with added tags presenting risk score and triggered risk rules from Recorded Future. Malware and" + " Threat Actors related to the enriched indicator in Recorded Future is matched against MISP's galaxy clusters" + " and applied as galaxy tags. The custom enrichment object also includes a list of related indicators from" + " Recorded Future (IP's, domains, hashes, URL's and vulnerabilities) added as additional attributes." + ), + "references": ["https://www.recordedfuture.com/"], + "input": ( + "A MISP attribute of one of the following types: ip, ip-src, ip-dst, domain, hostname, md5, sha1, sha256, uri," + " url, vulnerability, weakness." + ), + "output": ( + "A MISP object containing a copy of the enriched attribute with added tags from Recorded Future and a list of" + " new attributes related to the enriched attribute." + ), } moduleconfig = ["token", "proxy_host", "proxy_port", "proxy_username", "proxy_password"] @@ -87,9 +95,7 @@ def get(self, url: str, headers: dict = None) -> requests.Response: """General get method with proxy error handling.""" try: timeout = 7 if self.proxies else None - response = self.session.get( - url, headers=headers, proxies=self.proxies, timeout=timeout - ) + response = self.session.get(url, headers=headers, proxies=self.proxies, timeout=timeout) response.raise_for_status() return response except (ConnectTimeout, ProxyError, InvalidURL) as error: @@ -172,13 +178,9 @@ def pull_galaxy_cluster(self, related_type: str) -> None: try: response = GLOBAL_REQUEST_HANDLER.get(source) name = source.split("/")[-1].split(".")[0] - self.galaxy_clusters.setdefault(related_type, {}).update( - {name: response.json()} - ) + self.galaxy_clusters.setdefault(related_type, {}).update({name: response.json()}) except ConnectionError as error: - LOGGER.warning( - f"pull_galaxy_cluster failed for source: {source}, with error: {error}." - ) + LOGGER.warning(f"pull_galaxy_cluster failed for source: {source}, with error: {error}.") def find_galaxy_match(self, indicator: str, related_type: str) -> str: """Searches the clusters of the related_type for a match with the indicator. @@ -187,9 +189,7 @@ def find_galaxy_match(self, indicator: str, related_type: str) -> str: self.pull_galaxy_cluster(related_type) for cluster_name, cluster in self.galaxy_clusters.get(related_type, {}).items(): for value in cluster["values"]: - if indicator in value.get("meta", {}).get( - "synonyms", "" - ) or indicator in value.get("value", ""): + if indicator in value.get("meta", {}).get("synonyms", "") or indicator in value.get("value", ""): value = value["value"] return f'misp-galaxy:{cluster_name}="{value}"' return "" @@ -252,22 +252,15 @@ def __init__(self, attribute_props: dict): self.enrichment_object.template_uuid = "cbe0ffda-75e5-4c49-833f-093f057652ba" self.enrichment_object.template_id = "1" self.enrichment_object.description = "Recorded Future Enrichment" - setattr(self.enrichment_object, 'meta-category', 'network') - description = ( - "An object containing the enriched attribute and " - "related entities from Recorded Future." - ) - self.enrichment_object.from_dict( - **{"meta-category": "misc", "description": description, "distribution": 0} - ) + setattr(self.enrichment_object, "meta-category", "network") + description = "An object containing the enriched attribute and related entities from Recorded Future." + self.enrichment_object.from_dict(**{"meta-category": "misc", "description": description, "distribution": 0}) # Create a copy of enriched attribute to add tags to temp_attr = MISPAttribute() temp_attr.from_dict(**attribute_props) self.enriched_attribute = MISPAttribute() - self.enriched_attribute.from_dict( - **{"value": temp_attr.value, "type": temp_attr.type, "distribution": 0} - ) + self.enriched_attribute.from_dict(**{"value": temp_attr.value, "type": temp_attr.type, "distribution": 0}) self.related_attributes: List[Tuple[str, MISPAttribute]] = [] self.color_picker = RFColors() @@ -322,9 +315,7 @@ def enrich(self) -> None: # since RF do not support enriching ip addresses with port if self.enriched_attribute.type in ["ip-src|port", "ip-dst|port"]: enriched_attribute_value = enriched_attribute_value.split("|")[0] - json_response = GLOBAL_REQUEST_HANDLER.rf_lookup( - category, enriched_attribute_value - ) + json_response = GLOBAL_REQUEST_HANDLER.rf_lookup(category, enriched_attribute_value) response = json.loads(json_response.content) try: @@ -356,9 +347,7 @@ def enrich(self) -> None: entity_type = sec_list["type"]["name"] for entity in sec_list["entities"]: if entity_type in self.galaxy_tag_types: - galaxy = self.galaxy_finder.find_galaxy_match( - entity["name"], entity_type - ) + galaxy = self.galaxy_finder.find_galaxy_match(entity["name"], entity_type) if galaxy and galaxy not in galaxy_tags: galaxy_tags.append(galaxy) else: @@ -384,9 +373,7 @@ def enrich(self) -> None: # because there can be a huge list of related entities if int(related["count"]) > 4: indicator = related["entity"]["name"] - galaxy = self.galaxy_finder.find_galaxy_match( - indicator, related_type - ) + galaxy = self.galaxy_finder.find_galaxy_match(indicator, related_type) # Handle deduplication of galaxy tags if galaxy and galaxy not in galaxy_tags: galaxy_tags.append(galaxy) @@ -446,9 +433,7 @@ def get_output_type(self, related_type: str, indicator: str) -> str: def get_results(self) -> dict: """Build and return the enrichment results.""" - self.enrichment_object.add_attribute( - "Enriched attribute", **self.enriched_attribute - ) + self.enrichment_object.add_attribute("Enriched attribute", **self.enriched_attribute) for related_type, attribute in self.related_attributes: self.enrichment_object.add_attribute(related_type, **attribute) self.event.add_object(**self.enrichment_object) @@ -469,8 +454,7 @@ def get_proxy_settings(config: dict) -> Optional[Dict[str, str]]: if host: if not port: misperrors["error"] = ( - "The recordedfuture_proxy_host config is set, " - "please also set the recordedfuture_proxy_port." + "The recordedfuture_proxy_host config is set, please also set the recordedfuture_proxy_port." ) raise KeyError parsed = urlparse(host) @@ -509,9 +493,7 @@ def handler(q=False): else: misperrors["error"] = "Missing Recorded Future token." return misperrors - if not request.get("attribute") or not check_input_attribute( - request["attribute"], requirements=("type", "value") - ): + if not request.get("attribute") or not check_input_attribute(request["attribute"], requirements=("type", "value")): return {"error": f"{standard_error_message}, {checking_error}."} if request["attribute"]["type"] not in mispattributes["input"]: return {"error": "Unsupported attribute type."} diff --git a/misp_modules/modules/expansion/reversedns.py b/misp_modules/modules/expansion/reversedns.py index 5a2adc4ab..186968ea4 100644 --- a/misp_modules/modules/expansion/reversedns.py +++ b/misp_modules/modules/expansion/reversedns.py @@ -1,40 +1,46 @@ import json -from dns import reversename, resolver, exception -misperrors = {'error': 'Error'} -mispattributes = {'input': ['ip-src', 'ip-dst', 'domain|ip'], 'output': ['hostname']} +from dns import exception, resolver, reversename + +misperrors = {"error": "Error"} +mispattributes = {"input": ["ip-src", "ip-dst", "domain|ip"], "output": ["hostname"]} # possible module-types: 'expansion', 'hover' or both moduleinfo = { - 'version': '0.1', - 'author': 'Andreas Muehlemann', - 'description': 'Simple Reverse DNS expansion service to resolve reverse DNS from MISP attributes.', - 'module-type': ['expansion', 'hover'], - 'name': 'Reverse DNS', - 'logo': '', - 'requirements': ['DNS python library'], - 'features': 'The module takes an IP address as input and tries to find the hostname this IP address is resolved into.\n\nThe address of the DNS resolver to use is also configurable, but if no configuration is set, we use the Google public DNS address (8.8.8.8).\n\nPlease note that composite MISP attributes containing IP addresses are supported as well.', - 'references': [], - 'input': 'An IP address attribute.', - 'output': 'Hostname attribute the input is resolved into.', + "version": "0.1", + "author": "Andreas Muehlemann", + "description": "Simple Reverse DNS expansion service to resolve reverse DNS from MISP attributes.", + "module-type": ["expansion", "hover"], + "name": "Reverse DNS", + "logo": "", + "requirements": ["DNS python library"], + "features": ( + "The module takes an IP address as input and tries to find the hostname this IP address is resolved" + " into.\n\nThe address of the DNS resolver to use is also configurable, but if no configuration is set, we use" + " the Google public DNS address (8.8.8.8).\n\nPlease note that composite MISP attributes containing IP" + " addresses are supported as well." + ), + "references": [], + "input": "An IP address attribute.", + "output": "Hostname attribute the input is resolved into.", } # config fields that your code expects from the site admin -moduleconfig = ['nameserver'] +moduleconfig = ["nameserver"] def handler(q=False): if q is False: return False request = json.loads(q) - if request.get('ip-dst'): - toquery = request['ip-dst'] - elif request.get('ip-src'): - toquery = request['ip-src'] - elif request.get('ip'): - toquery = request['ip'] - elif request.get('domain|ip'): - toquery = request['domain|ip'].split('|')[1] + if request.get("ip-dst"): + toquery = request["ip-dst"] + elif request.get("ip-src"): + toquery = request["ip-src"] + elif request.get("ip"): + toquery = request["ip"] + elif request.get("domain|ip"): + toquery = request["domain|ip"].split("|")[1] else: return False @@ -43,27 +49,27 @@ def handler(q=False): r = resolver.Resolver() r.timeout = 2 r.lifetime = 2 - if request.get('config'): - if request['config'].get('nameserver'): + if request.get("config"): + if request["config"].get("nameserver"): nameservers = [] - nameservers.append(request['config'].get('nameserver')) + nameservers.append(request["config"].get("nameserver")) r.nameservers = nameservers else: - r.nameservers = ['8.8.8.8'] + r.nameservers = ["8.8.8.8"] try: - answer = r.resolve(revname, 'PTR') + answer = r.resolve(revname, "PTR") except resolver.NXDOMAIN: - misperrors['error'] = "NXDOMAIN" + misperrors["error"] = "NXDOMAIN" return misperrors except exception.Timeout: - misperrors['error'] = "Timeout" + misperrors["error"] = "Timeout" return misperrors except Exception: - misperrors['error'] = "DNS resolving error" + misperrors["error"] = "DNS resolving error" return misperrors - r = {'results': [{'types': mispattributes['output'],'values':[str(answer[0])]}]} + r = {"results": [{"types": mispattributes["output"], "values": [str(answer[0])]}]} return r @@ -72,5 +78,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index ae251c57b..8fbd4670d 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -3,43 +3,61 @@ import sys import time -from dnstrails import APIError -from dnstrails import DnsTrails +from dnstrails import APIError, DnsTrails -log = logging.getLogger('dnstrails') +log = logging.getLogger("dnstrails") log.setLevel(logging.DEBUG) ch = logging.StreamHandler(sys.stdout) ch.setLevel(logging.DEBUG) -formatter = logging.Formatter( - '%(asctime)s - %(name)s - %(levelname)s - %(message)s') +formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") ch.setFormatter(formatter) log.addHandler(ch) -misperrors = {'error': 'Error'} +misperrors = {"error": "Error"} mispattributes = { - 'input': ['hostname', 'domain', 'ip-src', 'ip-dst'], - 'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'dns-soa-email', - 'whois-registrant-email', 'whois-registrant-phone', - 'whois-registrant-name', - 'whois-registrar', 'whois-creation-date', 'domain'] + "input": ["hostname", "domain", "ip-src", "ip-dst"], + "output": [ + "hostname", + "domain", + "ip-src", + "ip-dst", + "dns-soa-email", + "whois-registrant-email", + "whois-registrant-phone", + "whois-registrant-name", + "whois-registrar", + "whois-creation-date", + "domain", + ], } moduleinfo = { - 'version': '1', - 'author': 'Sebastien Larinier @sebdraven', - 'description': 'An expansion modules for SecurityTrails.', - 'module-type': ['expansion', 'hover'], - 'name': 'SecurityTrails Lookup', - 'logo': 'securitytrails.png', - 'requirements': ['dnstrails python library', 'An access to the SecurityTrails API (apikey)'], - 'features': 'The module takes a domain, hostname or IP address attribute as input and queries the SecurityTrails API with it.\n\nMultiple parsing operations are then processed on the result of the query to extract a much information as possible.\n\nFrom this data extracted are then mapped MISP attributes.', - 'references': ['https://securitytrails.com/'], - 'input': 'A domain, hostname or IP address attribute.', - 'output': 'MISP attributes resulting from the query on SecurityTrails API, included in the following list:\n- hostname\n- domain\n- ip-src\n- ip-dst\n- dns-soa-email\n- whois-registrant-email\n- whois-registrant-phone\n- whois-registrant-name\n- whois-registrar\n- whois-creation-date\n- domain', + "version": "1", + "author": "Sebastien Larinier @sebdraven", + "description": "An expansion modules for SecurityTrails.", + "module-type": ["expansion", "hover"], + "name": "SecurityTrails Lookup", + "logo": "securitytrails.png", + "requirements": [ + "dnstrails python library", + "An access to the SecurityTrails API (apikey)", + ], + "features": ( + "The module takes a domain, hostname or IP address attribute as input and queries the SecurityTrails API with" + " it.\n\nMultiple parsing operations are then processed on the result of the query to extract a much" + " information as possible.\n\nFrom this data extracted are then mapped MISP attributes." + ), + "references": ["https://securitytrails.com/"], + "input": "A domain, hostname or IP address attribute.", + "output": ( + "MISP attributes resulting from the query on SecurityTrails API, included in the following list:\n- hostname\n-" + " domain\n- ip-src\n- ip-dst\n- dns-soa-email\n- whois-registrant-email\n- whois-registrant-phone\n-" + " whois-registrant-name\n- whois-registrar\n- whois-creation-date\n- domain" + ), } # config fields that your code expects from the site admin -moduleconfig = ['apikey'] +moduleconfig = ["apikey"] def handler(q=False): @@ -47,29 +65,29 @@ def handler(q=False): request = json.loads(q) - if not request.get('config') or not (request['config'].get('apikey')): - misperrors['error'] = 'SecurityTrails authentication is missing' + if not request.get("config") or not (request["config"].get("apikey")): + misperrors["error"] = "SecurityTrails authentication is missing" return misperrors - api = DnsTrails(request['config'].get('apikey')) + api = DnsTrails(request["config"].get("apikey")) if not api: - misperrors['error'] = 'SecurityTrails Error instance api' + misperrors["error"] = "SecurityTrails Error instance api" return misperrors - if request.get('ip-src'): - ip = request['ip-src'] + if request.get("ip-src"): + ip = request["ip-src"] return handle_ip(api, ip, misperrors) - elif request.get('ip-dst'): - ip = request['ip-dst'] + elif request.get("ip-dst"): + ip = request["ip-dst"] return handle_ip(api, ip, misperrors) - elif request.get('domain'): - domain = request['domain'] + elif request.get("domain"): + domain = request["domain"] return handle_domain(api, domain, misperrors) - elif request.get('hostname'): - hostname = request['hostname'] + elif request.get("hostname"): + hostname = request["hostname"] return handle_domain(api, hostname, misperrors) else: - misperrors['error'] = "Unsupported attributes types" + misperrors["error"] = "Unsupported attributes types" return misperrors else: return False @@ -82,9 +100,9 @@ def handle_domain(api, domain, misperrors): if status_ok: if r: - result_filtered['results'].extend(r) + result_filtered["results"].extend(r) else: - misperrors['error'] = misperrors['error'] + ' Error DNS result' + misperrors["error"] = misperrors["error"] + " Error DNS result" return misperrors time.sleep(1) @@ -92,9 +110,9 @@ def handle_domain(api, domain, misperrors): if status_ok: if r: - result_filtered['results'].extend(r) + result_filtered["results"].extend(r) else: - misperrors['error'] = misperrors['error'] + ' Error subdomains result' + misperrors["error"] = misperrors["error"] + " Error subdomains result" return misperrors time.sleep(1) @@ -102,16 +120,16 @@ def handle_domain(api, domain, misperrors): if status_ok: if r: - result_filtered['results'].extend(r) + result_filtered["results"].extend(r) time.sleep(1) r, status_ok = expand_history_ipv4_ipv6(api, domain) if status_ok: if r: - result_filtered['results'].extend(r) + result_filtered["results"].extend(r) else: - misperrors['error'] = misperrors['error'] + ' Error history ipv4' + misperrors["error"] = misperrors["error"] + " Error history ipv4" return misperrors time.sleep(1) @@ -120,18 +138,18 @@ def handle_domain(api, domain, misperrors): if status_ok: if r: - result_filtered['results'].extend(r) + result_filtered["results"].extend(r) else: - misperrors['error'] = misperrors['error'] + ' Error in expand History DNS' + misperrors["error"] = misperrors["error"] + " Error in expand History DNS" return misperrors r, status_ok = expand_history_whois(api, domain) if status_ok: if r: - result_filtered['results'].extend(r) + result_filtered["results"].extend(r) else: - misperrors['error'] = misperrors['error'] + ' Error in expand History Whois' + misperrors["error"] = misperrors["error"] + " Error in expand History Whois" return misperrors return result_filtered @@ -144,9 +162,9 @@ def handle_ip(api, ip, misperrors): if status_ok: if r: - result_filtered['results'].extend(r) + result_filtered["results"].extend(r) else: - misperrors['error'] += ' Error in expand searching domain' + misperrors["error"] += " Error in expand searching domain" return misperrors return result_filtered @@ -164,83 +182,91 @@ def expand_domain_info(api, misperror, domain): try: results = api.domain(domain) except APIError as e: - misperrors['error'] = e.value + misperrors["error"] = e.value return [], False if results: status_ok = True - if 'current_dns' in results: - if 'values' in results['current_dns']['ns']: - ns_servers = [ns_entry['nameserver'] for ns_entry in - results['current_dns']['ns']['values'] - if 'nameserver' in ns_entry] - if 'values' in results['current_dns']['a']: - list_ipv4 = [a_entry['ip'] for a_entry in - results['current_dns']['a']['values'] if - 'ip' in a_entry] - - if 'values' in results['current_dns']['aaaa']: - list_ipv6 = [ipv6_entry['ipv6'] for ipv6_entry in - results['current_dns']['aaaa']['values'] if - 'ipv6' in ipv6_entry] - - if 'values' in results['current_dns']['mx']: - servers_mx = [mx_entry['hostname'] for mx_entry in - results['current_dns']['mx']['values'] if - 'hostname' in mx_entry] - if 'values' in results['current_dns']['soa']: - soa_hostnames = [soa_entry['email'] for soa_entry in - results['current_dns']['soa']['values'] if - 'email' in soa_entry] + if "current_dns" in results: + if "values" in results["current_dns"]["ns"]: + ns_servers = [ + ns_entry["nameserver"] + for ns_entry in results["current_dns"]["ns"]["values"] + if "nameserver" in ns_entry + ] + if "values" in results["current_dns"]["a"]: + list_ipv4 = [a_entry["ip"] for a_entry in results["current_dns"]["a"]["values"] if "ip" in a_entry] + + if "values" in results["current_dns"]["aaaa"]: + list_ipv6 = [ + ipv6_entry["ipv6"] + for ipv6_entry in results["current_dns"]["aaaa"]["values"] + if "ipv6" in ipv6_entry + ] + + if "values" in results["current_dns"]["mx"]: + servers_mx = [ + mx_entry["hostname"] + for mx_entry in results["current_dns"]["mx"]["values"] + if "hostname" in mx_entry + ] + if "values" in results["current_dns"]["soa"]: + soa_hostnames = [ + soa_entry["email"] for soa_entry in results["current_dns"]["soa"]["values"] if "email" in soa_entry + ] if ns_servers: - r.append({'types': ['domain'], - 'values': ns_servers, - 'categories': ['Network activity'], - 'comment': 'List of name servers of %s first seen %s ' % - (domain, - results['current_dns']['ns']['first_seen']) - }) + r.append( + { + "types": ["domain"], + "values": ns_servers, + "categories": ["Network activity"], + "comment": ( + "List of name servers of %s first seen %s " + % (domain, results["current_dns"]["ns"]["first_seen"]) + ), + } + ) if list_ipv4: - r.append({'types': ['domain|ip'], - 'values': ['%s|%s' % (domain, ipv4) for ipv4 in - list_ipv4], - 'categories': ['Network activity'], - - 'comment': ' List ipv4 of %s first seen %s' % - (domain, - results['current_dns']['a']['first_seen']) - - }) + r.append( + { + "types": ["domain|ip"], + "values": ["%s|%s" % (domain, ipv4) for ipv4 in list_ipv4], + "categories": ["Network activity"], + "comment": " List ipv4 of %s first seen %s" % (domain, results["current_dns"]["a"]["first_seen"]), + } + ) if list_ipv6: - r.append({'types': ['domain|ip'], - 'values': ['%s|%s' % (domain, ipv6) for ipv6 in - list_ipv6], - 'categories': ['Network activity'], - 'comment': ' List ipv6 of %s first seen %s' % - (domain, - results['current_dns']['aaaa']['first_seen']) - - }) + r.append( + { + "types": ["domain|ip"], + "values": ["%s|%s" % (domain, ipv6) for ipv6 in list_ipv6], + "categories": ["Network activity"], + "comment": ( + " List ipv6 of %s first seen %s" % (domain, results["current_dns"]["aaaa"]["first_seen"]) + ), + } + ) if servers_mx: - r.append({'types': ['domain'], - 'values': servers_mx, - 'categories': ['Network activity'], - 'comment': ' List mx of %s first seen %s' % - (domain, - results['current_dns']['mx']['first_seen']) - - }) + r.append( + { + "types": ["domain"], + "values": servers_mx, + "categories": ["Network activity"], + "comment": " List mx of %s first seen %s" % (domain, results["current_dns"]["mx"]["first_seen"]), + } + ) if soa_hostnames: - r.append({'types': ['domain'], - 'values': soa_hostnames, - 'categories': ['Network activity'], - 'comment': ' List soa of %s first seen %s' % - (domain, - results['current_dns']['soa']['first_seen']) - }) + r.append( + { + "types": ["domain"], + "values": soa_hostnames, + "categories": ["Network activity"], + "comment": " List soa of %s first seen %s" % (domain, results["current_dns"]["soa"]["first_seen"]), + } + ) return r, status_ok @@ -254,18 +280,17 @@ def expand_subdomains(api, domain): if results: status_ok = True - if 'subdomains' in results: - r.append({ - 'types': ['domain'], - 'values': ['%s.%s' % (sub, domain) - for sub in results['subdomains']], - 'categories': ['Network activity'], - 'comment': 'subdomains of %s' % domain - } - + if "subdomains" in results: + r.append( + { + "types": ["domain"], + "values": ["%s.%s" % (sub, domain) for sub in results["subdomains"]], + "categories": ["Network activity"], + "comment": "subdomains of %s" % domain, + } ) except APIError as e: - misperrors['error'] = e.value + misperrors["error"] = e.value return [], False return r, status_ok @@ -283,63 +308,58 @@ def expand_whois(api, domain): item_registrant = __select_registrant_item(results) if item_registrant: - if 'email' in item_registrant[0]: + if "email" in item_registrant[0]: r.append( { - 'types': ['whois-registrant-email'], - 'values': [item_registrant[0]['email']], - 'categories': ['Attribution'], - 'comment': 'Whois information of %s by securitytrails' - % domain + "types": ["whois-registrant-email"], + "values": [item_registrant[0]["email"]], + "categories": ["Attribution"], + "comment": "Whois information of %s by securitytrails" % domain, } ) - if 'telephone' in item_registrant[0]: + if "telephone" in item_registrant[0]: r.append( { - 'types': ['whois-registrant-phone'], - 'values': [item_registrant[0]['telephone']], - 'categories': ['Attribution'], - 'comment': 'Whois information of %s by securitytrails' - % domain + "types": ["whois-registrant-phone"], + "values": [item_registrant[0]["telephone"]], + "categories": ["Attribution"], + "comment": "Whois information of %s by securitytrails" % domain, } ) - if 'name' in item_registrant[0]: + if "name" in item_registrant[0]: r.append( { - 'types': ['whois-registrant-name'], - 'values': [item_registrant[0]['name']], - 'categories': ['Attribution'], - 'comment': 'Whois information of %s by securitytrails' - % domain + "types": ["whois-registrant-name"], + "values": [item_registrant[0]["name"]], + "categories": ["Attribution"], + "comment": "Whois information of %s by securitytrails" % domain, } ) - if 'registrarName' in item_registrant[0]: + if "registrarName" in item_registrant[0]: r.append( { - 'types': ['whois-registrar'], - 'values': [item_registrant[0]['registrarName']], - 'categories': ['Attribution'], - 'comment': 'Whois information of %s by securitytrails' - % domain + "types": ["whois-registrar"], + "values": [item_registrant[0]["registrarName"]], + "categories": ["Attribution"], + "comment": "Whois information of %s by securitytrails" % domain, } ) - if 'createdDate' in item_registrant[0]: + if "createdDate" in item_registrant[0]: r.append( { - 'types': ['whois-creation-date'], - 'values': [item_registrant[0]['createdDate']], - 'categories': ['Attribution'], - 'comment': 'Whois information of %s by securitytrails' - % domain + "types": ["whois-creation-date"], + "values": [item_registrant[0]["createdDate"]], + "categories": ["Attribution"], + "comment": "Whois information of %s by securitytrails" % domain, } ) except APIError as e: - misperrors['error'] = e.value + misperrors["error"] = e.value return [], False return r, status_ok @@ -361,10 +381,10 @@ def expand_history_ipv4_ipv6(api, domain): if results: status_ok = True - r.extend(__history_ip(results, domain, type_ip='ipv6')) + r.extend(__history_ip(results, domain, type_ip="ipv6")) except APIError as e: - misperrors['error'] = e.value + misperrors["error"] = e.value return [], False return r, status_ok @@ -378,14 +398,14 @@ def expand_history_dns(api, domain): results = api.history_dns_ns(domain) if results: - r.extend(__history_dns(results, domain, 'nameserver', 'ns')) + r.extend(__history_dns(results, domain, "nameserver", "ns")) time.sleep(1) results = api.history_dns_soa(domain) if results: - r.extend(__history_dns(results, domain, 'email', 'soa')) + r.extend(__history_dns(results, domain, "email", "soa")) time.sleep(1) @@ -393,10 +413,10 @@ def expand_history_dns(api, domain): if results: status_ok = True - r.extend(__history_dns(results, domain, 'host', 'mx')) + r.extend(__history_dns(results, domain, "host", "mx")) except APIError as e: - misperrors['error'] = e.value + misperrors["error"] = e.value return [], False status_ok = True @@ -412,72 +432,72 @@ def expand_history_whois(api, domain): if results: - if 'items' in results['result']: - for item in results['result']['items']: + if "items" in results["result"]: + for item in results["result"]["items"]: item_registrant = __select_registrant_item(item) r.append( { - 'types': ['domain'], - 'values': item['nameServers'], - 'categories': ['Network activity'], - 'comment': 'Whois history Name Servers of %s ' - 'Status: %s ' % ( - domain, ' '.join(item['status'])) - + "types": ["domain"], + "values": item["nameServers"], + "categories": ["Network activity"], + "comment": ( + "Whois history Name Servers of %s Status: %s " % (domain, " ".join(item["status"])) + ), } ) if item_registrant: - if 'email' in item_registrant[0]: + if "email" in item_registrant[0]: r.append( { - 'types': ['whois-registrant-email'], - 'values': [item_registrant[0]['email']], - 'categories': ['Attribution'], - 'comment': 'Whois history registrant email of %s' - 'Status: %s' % ( - domain, - ' '.join(item['status'])) + "types": ["whois-registrant-email"], + "values": [item_registrant[0]["email"]], + "categories": ["Attribution"], + "comment": ( + "Whois history registrant email of %sStatus: %s" + % (domain, " ".join(item["status"])) + ), } ) - if 'telephone' in item_registrant[0]: + if "telephone" in item_registrant[0]: r.append( { - 'types': ['whois-registrant-phone'], - 'values': [item_registrant[0]['telephone']], - 'categories': ['Attribution'], - 'comment': 'Whois history registrant phone of %s' - 'Status: %s' % ( - domain, - ' '.join(item['status'])) + "types": ["whois-registrant-phone"], + "values": [item_registrant[0]["telephone"]], + "categories": ["Attribution"], + "comment": ( + "Whois history registrant phone of %sStatus: %s" + % (domain, " ".join(item["status"])) + ), } ) except APIError as e: - misperrors['error'] = e.value + misperrors["error"] = e.value return [], False status_ok = True return r, status_ok -def __history_ip(results, domain, type_ip='ip'): +def __history_ip(results, domain, type_ip="ip"): r = [] - if 'records' in results: - for record in results['records']: - if 'values' in record: - for item in record['values']: + if "records" in results: + for record in results["records"]: + if "values" in record: + for item in record["values"]: r.append( - {'types': ['domain|ip'], - 'values': ['%s|%s' % (domain, item[type_ip])], - 'categories': ['Network activity'], - 'comment': 'History IP on securitytrails %s ' - 'last seen: %s first seen: %s' % - (domain, record['last_seen'], - record['first_seen']) - } + { + "types": ["domain|ip"], + "values": ["%s|%s" % (domain, item[type_ip])], + "categories": ["Network activity"], + "comment": ( + "History IP on securitytrails %s last seen: %s first seen: %s" + % (domain, record["last_seen"], record["first_seen"]) + ), + } ) return r @@ -486,31 +506,45 @@ def __history_ip(results, domain, type_ip='ip'): def __history_dns(results, domain, type_serv, service): r = [] - if 'records' in results: - for record in results['records']: - if 'values' in record: - values = record['values'] + if "records" in results: + for record in results["records"]: + if "values" in record: + values = record["values"] if type(values) is list: - for item in record['values']: + for item in record["values"]: r.append( - {'types': ['domain|ip'], - 'values': [item[type_serv]], - 'categories': ['Network activity'], - 'comment': 'history %s of %s last seen: %s first seen: %s' % - (service, domain, record['last_seen'], - record['first_seen']) - } + { + "types": ["domain|ip"], + "values": [item[type_serv]], + "categories": ["Network activity"], + "comment": ( + "history %s of %s last seen: %s first seen: %s" + % ( + service, + domain, + record["last_seen"], + record["first_seen"], + ) + ), + } ) else: r.append( - {'types': ['domain|ip'], - 'values': [values[type_serv]], - 'categories': ['Network activity'], - 'comment': 'history %s of %s last seen: %s first seen: %s' % - (service, domain, record['last_seen'], - record['first_seen']) - } + { + "types": ["domain|ip"], + "values": [values[type_serv]], + "categories": ["Network activity"], + "comment": ( + "history %s of %s last seen: %s first seen: %s" + % ( + service, + domain, + record["last_seen"], + record["first_seen"], + ) + ), + } ) return r @@ -523,27 +557,25 @@ def expand_searching_domain(api, ip): results = api.searching_domains(ipv4=ip) if results: - if 'records' in results: - res = [(r['host_provider'], r['hostname'], r['whois']) - for r in results['records']] + if "records" in results: + res = [(r["host_provider"], r["hostname"], r["whois"]) for r in results["records"]] for host_provider, hostname, whois in res: - comment = 'domain for %s by %s' % (ip, host_provider[0]) - if whois['registrar']: - comment = comment + ' registrar %s' % whois['registrar'] + comment = "domain for %s by %s" % (ip, host_provider[0]) + if whois["registrar"]: + comment = comment + " registrar %s" % whois["registrar"] r.append( { - 'types': ['domain'], - 'category': ['Network activity'], - 'values': [hostname], - 'comment': comment - + "types": ["domain"], + "category": ["Network activity"], + "values": [hostname], + "comment": comment, } ) status_ok = True except APIError as e: - misperrors['error'] = e.value + misperrors["error"] = e.value return [], False return r, status_ok @@ -554,18 +586,16 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo def __select_registrant_item(entry): res = None - if 'contacts' in entry: - res = list(filter(lambda x: x['type'] == 'registrant', - entry['contacts'])) + if "contacts" in entry: + res = list(filter(lambda x: x["type"] == "registrant", entry["contacts"])) - if 'contact' in entry: - res = list(filter(lambda x: x['type'] == 'registrant', - entry['contact'])) + if "contact" in entry: + res = list(filter(lambda x: x["type"] == "registrant", entry["contact"])) return res diff --git a/misp_modules/modules/expansion/shodan.py b/misp_modules/modules/expansion/shodan.py index 9c3ab4f42..f390e04a6 100755 --- a/misp_modules/modules/expansion/shodan.py +++ b/misp_modules/modules/expansion/shodan.py @@ -1,70 +1,70 @@ # -*- coding: utf-8 -*- import json -try: - import shodan -except ImportError: - print("shodan module not installed.") -from . import check_input_attribute, standard_error_message from datetime import datetime + +import shodan from pymisp import MISPAttribute, MISPEvent, MISPObject -misperrors = {'error': 'Error'} -mispattributes = {'input': ['ip-src', 'ip-dst'], - 'format': 'misp_standard'} +from . import check_input_attribute, standard_error_message + +misperrors = {"error": "Error"} +mispattributes = {"input": ["ip-src", "ip-dst"], "format": "misp_standard"} moduleinfo = { - 'version': '0.2', - 'author': 'Raphaël Vinot', - 'description': 'Module to query on Shodan.', - 'module-type': ['expansion'], - 'name': 'Shodan Lookup', - 'logo': 'shodan.png', - 'requirements': ['shodan python library', 'An access to the Shodan API (apikey)'], - 'features': 'The module takes an IP address as input and queries the Shodan API to get some additional data about it.', - 'references': ['https://www.shodan.io/'], - 'input': 'An IP address MISP attribute.', - 'output': 'Text with additional data about the input, resulting from the query on Shodan.', + "version": "0.2", + "author": "Raphaël Vinot", + "description": "Module to query on Shodan.", + "module-type": ["expansion"], + "name": "Shodan Lookup", + "logo": "shodan.png", + "requirements": ["shodan python library", "An access to the Shodan API (apikey)"], + "features": ( + "The module takes an IP address as input and queries the Shodan API to get some additional data about it." + ), + "references": ["https://www.shodan.io/"], + "input": "An IP address MISP attribute.", + "output": "Text with additional data about the input, resulting from the query on Shodan.", } -moduleconfig = ['apikey'] +moduleconfig = ["apikey"] -class ShodanParser(): +class ShodanParser: def __init__(self, attribute): self.misp_event = MISPEvent() self.attribute = MISPAttribute() self.attribute.from_dict(**attribute) self.misp_event.add_attribute(**self.attribute) self.ip_address_mapping = { - 'asn': {'type': 'AS', 'object_relation': 'asn'}, - 'city': {'type': 'text', 'object_relation': 'city'}, - 'country_code': {'type': 'text', 'object_relation': 'country-code'}, - 'country_name': {'type': 'text', 'object_relation': 'country'}, - 'isp': {'type': 'text', 'object_relation': 'ISP'}, - 'latitude': {'type': 'float', 'object_relation': 'latitude'}, - 'longitude': {'type': 'float', 'object_relation': 'longitude'}, - 'org': {'type': 'text', 'object_relation': 'organization'}, - 'postal_code': {'type': 'text', 'object_relation': 'zipcode'}, - 'region_code': {'type': 'text', 'object_relation': 'region-code'} + "asn": {"type": "AS", "object_relation": "asn"}, + "city": {"type": "text", "object_relation": "city"}, + "country_code": {"type": "text", "object_relation": "country-code"}, + "country_name": {"type": "text", "object_relation": "country"}, + "isp": {"type": "text", "object_relation": "ISP"}, + "latitude": {"type": "float", "object_relation": "latitude"}, + "longitude": {"type": "float", "object_relation": "longitude"}, + "org": {"type": "text", "object_relation": "organization"}, + "postal_code": {"type": "text", "object_relation": "zipcode"}, + "region_code": {"type": "text", "object_relation": "region-code"}, } self.ip_port_mapping = { - 'domains': {'type': 'domain', 'object_relation': 'domain'}, - 'hostnames': {'type': 'hostname', 'object_relation': 'hostname'} + "domains": {"type": "domain", "object_relation": "domain"}, + "hostnames": {"type": "hostname", "object_relation": "hostname"}, } self.vulnerability_mapping = { - 'cvss': {'type': 'float', 'object_relation': 'cvss-score'}, - 'summary': {'type': 'text', 'object_relation': 'summary'} + "cvss": {"type": "float", "object_relation": "cvss-score"}, + "summary": {"type": "text", "object_relation": "summary"}, } self.x509_mapping = { - 'bits': {'type': 'text', 'object_relation': 'pubkey-info-size'}, - 'expires': {'type': 'datetime', 'object_relation': 'validity-not-after'}, - 'issued': {'type': 'datetime', 'object_relation': 'validity-not-before'}, - 'issuer': {'type': 'text', 'object_relation': 'issuer'}, - 'serial': {'type': 'text', 'object_relation': 'serial-number'}, - 'sig_alg': {'type': 'text', 'object_relation': 'signature_algorithm'}, - 'subject': {'type': 'text', 'object_relation': 'subject'}, - 'type': {'type': 'text', 'object_relation': 'pubkey-info-algorithm'}, - 'version': {'type': 'text', 'object_relation': 'version'} + "bits": {"type": "text", "object_relation": "pubkey-info-size"}, + "expires": {"type": "datetime", "object_relation": "validity-not-after"}, + "issued": {"type": "datetime", "object_relation": "validity-not-before"}, + "issuer": {"type": "text", "object_relation": "issuer"}, + "serial": {"type": "text", "object_relation": "serial-number"}, + "sig_alg": {"type": "text", "object_relation": "signature_algorithm"}, + "subject": {"type": "text", "object_relation": "subject"}, + "type": {"type": "text", "object_relation": "pubkey-info-algorithm"}, + "version": {"type": "text", "object_relation": "version"}, } def query_shodan(self, apikey): @@ -76,146 +76,143 @@ def query_shodan(self, apikey): ip_address_attributes = [] for feature, mapping in self.ip_address_mapping.items(): if query_results.get(feature): - attribute = {'value': query_results[feature]} + attribute = {"value": query_results[feature]} attribute.update(mapping) ip_address_attributes.append(attribute) if ip_address_attributes: - ip_address_object = MISPObject('ip-api-address') + ip_address_object = MISPObject("ip-api-address") for attribute in ip_address_attributes: ip_address_object.add_attribute(**attribute) - ip_address_object.add_reference(self.attribute.uuid, 'describes') + ip_address_object.add_reference(self.attribute.uuid, "describes") self.misp_event.add_object(ip_address_object) # Parse the hostnames / domains and ports associated with the IP address - if query_results.get('ports'): - ip_port_object = MISPObject('ip-port') + if query_results.get("ports"): + ip_port_object = MISPObject("ip-port") ip_port_object.add_attribute(**self._get_source_attribute()) - feature = self.attribute.type.split('-')[1] - for port in query_results['ports']: + feature = self.attribute.type.split("-")[1] + for port in query_results["ports"]: attribute = { - 'type': 'port', - 'object_relation': f'{feature}-port', - 'value': port + "type": "port", + "object_relation": f"{feature}-port", + "value": port, } ip_port_object.add_attribute(**attribute) for feature, mapping in self.ip_port_mapping.items(): for value in query_results.get(feature, []): - attribute = {'value': value} + attribute = {"value": value} attribute.update(mapping) ip_port_object.add_attribute(**attribute) - ip_port_object.add_reference(self.attribute.uuid, 'extends') + ip_port_object.add_reference(self.attribute.uuid, "extends") self.misp_event.add_object(ip_port_object) else: - if any(query_results.get(feature) for feature in ('domains', 'hostnames')): - domain_ip_object = MISPObject('domain-ip') + if any(query_results.get(feature) for feature in ("domains", "hostnames")): + domain_ip_object = MISPObject("domain-ip") domain_ip_object.add_attribute(**self._get_source_attribute()) - for feature in ('domains', 'hostnames'): + for feature in ("domains", "hostnames"): for value in query_results[feature]: attribute = { - 'type': 'domain', - 'object_relation': 'domain', - 'value': value + "type": "domain", + "object_relation": "domain", + "value": value, } domain_ip_object.add_attribute(**attribute) - domain_ip_object.add_reference(self.attribute.uuid, 'extends') + domain_ip_object.add_reference(self.attribute.uuid, "extends") self.misp_event.add_object(domain_ip_object) # Parse data within the "data" field - if query_results.get('vulns'): + if query_results.get("vulns"): vulnerabilities = {} - for data in query_results['data']: + for data in query_results["data"]: # Parse vulnerabilities - if data.get('vulns'): - for cve, vulnerability in data['vulns'].items(): + if data.get("vulns"): + for cve, vulnerability in data["vulns"].items(): if cve not in vulnerabilities: vulnerabilities[cve] = vulnerability # Also parse the certificates - if data.get('ssl'): - self._parse_cert(data['ssl']) + if data.get("ssl"): + self._parse_cert(data["ssl"]) for cve, vulnerability in vulnerabilities.items(): - vulnerability_object = MISPObject('vulnerability') - vulnerability_object.add_attribute(**{ - 'type': 'vulnerability', - 'object_relation': 'id', - 'value': cve - }) + vulnerability_object = MISPObject("vulnerability") + vulnerability_object.add_attribute(**{"type": "vulnerability", "object_relation": "id", "value": cve}) for feature, mapping in self.vulnerability_mapping.items(): if vulnerability.get(feature): - attribute = {'value': vulnerability[feature]} + attribute = {"value": vulnerability[feature]} attribute.update(mapping) vulnerability_object.add_attribute(**attribute) - if vulnerability.get('references'): - for reference in vulnerability['references']: - vulnerability_object.add_attribute(**{ - 'type': 'link', - 'object_relation': 'references', - 'value': reference - }) - vulnerability_object.add_reference(self.attribute.uuid, 'vulnerability-of') + if vulnerability.get("references"): + for reference in vulnerability["references"]: + vulnerability_object.add_attribute( + **{ + "type": "link", + "object_relation": "references", + "value": reference, + } + ) + vulnerability_object.add_reference(self.attribute.uuid, "vulnerability-of") self.misp_event.add_object(vulnerability_object) - for cve_id in query_results['vulns']: + for cve_id in query_results["vulns"]: if cve_id not in vulnerabilities: - attribute = { - 'type': 'vulnerability', - 'value': cve_id - } + attribute = {"type": "vulnerability", "value": cve_id} self.misp_event.add_attribute(**attribute) else: # We have no vulnerability data, we only check if we have # certificates within the "data" field - for data in query_results['data']: - if data.get('ssl'): - self._parse_cert(data['ssl']['cert']) + for data in query_results["data"]: + if data.get("ssl"): + self._parse_cert(data["ssl"]["cert"]) def get_result(self): event = json.loads(self.misp_event.to_json()) - results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} - return {'results': results} + results = {key: event[key] for key in ("Attribute", "Object") if (key in event and event[key])} + return {"results": results} # When we want to add the IP address information in objects such as the # domain-ip or ip-port objects referencing the input IP address attribute def _get_source_attribute(self): return { - 'type': self.attribute.type, - 'object_relation': self.attribute.type, - 'value': self.attribute.value + "type": self.attribute.type, + "object_relation": self.attribute.type, + "value": self.attribute.value, } def _parse_cert(self, certificate): - x509_object = MISPObject('x509') - for feature in ('serial', 'sig_alg', 'version'): + x509_object = MISPObject("x509") + for feature in ("serial", "sig_alg", "version"): if certificate.get(feature): - attribute = {'value': certificate[feature]} + attribute = {"value": certificate[feature]} attribute.update(self.x509_mapping[feature]) x509_object.add_attribute(**attribute) # Parse issuer and subject value - for feature in ('issuer', 'subject'): + for feature in ("issuer", "subject"): if certificate.get(feature): - attribute_value = (f'{identifier}={value}' for identifier, value in certificate[feature].items()) - attribute = {'value': f'/{"/".join(attribute_value)}'} + attribute_value = (f"{identifier}={value}" for identifier, value in certificate[feature].items()) + attribute = {"value": f'/{"/".join(attribute_value)}'} attribute.update(self.x509_mapping[feature]) x509_object.add_attribute(**attribute) # Parse datetime attributes - for feature in ('expires', 'issued'): + for feature in ("expires", "issued"): if certificate.get(feature): - attribute = {'value': datetime.strptime(certificate[feature], '%Y%m%d%H%M%SZ')} + attribute = {"value": datetime.strptime(certificate[feature], "%Y%m%d%H%M%SZ")} attribute.update(self.x509_mapping[feature]) x509_object.add_attribute(**attribute) # Parse fingerprints - if certificate.get('fingerprint'): - for hash_type, hash_value in certificate['fingerprint'].items(): - x509_object.add_attribute(**{ - 'type': f'x509-fingerprint-{hash_type}', - 'object_relation': f'x509-fingerprint-{hash_type}', - 'value': hash_value - }) + if certificate.get("fingerprint"): + for hash_type, hash_value in certificate["fingerprint"].items(): + x509_object.add_attribute( + **{ + "type": f"x509-fingerprint-{hash_type}", + "object_relation": f"x509-fingerprint-{hash_type}", + "value": hash_value, + } + ) # Parse public key related info - if certificate.get('pubkey'): - for feature, value in certificate['pubkey'].items(): - attribute = {'value': value} + if certificate.get("pubkey"): + for feature, value in certificate["pubkey"].items(): + attribute = {"value": value} attribute.update(self.x509_mapping[feature]) x509_object.add_attribute(**attribute) - x509_object.add_reference(self.attribute.uuid, 'identifies') + x509_object.add_reference(self.attribute.uuid, "identifies") self.misp_event.add_object(x509_object) @@ -223,15 +220,15 @@ def handler(q=False): if q is False: return False request = json.loads(q) - if not request.get('config', {}).get('apikey'): - return {'error': 'Shodan authentication is missing'} - if not request.get('attribute') or not check_input_attribute(request['attribute']): - return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} - attribute = request['attribute'] - if attribute['type'] not in mispattributes['input']: - return {'error': 'Unsupported attribute type.'} + if not request.get("config", {}).get("apikey"): + return {"error": "Shodan authentication is missing"} + if not request.get("attribute") or not check_input_attribute(request["attribute"]): + return {"error": f"{standard_error_message}, which should contain at least a type, a value and an uuid."} + attribute = request["attribute"] + if attribute["type"] not in mispattributes["input"]: + return {"error": "Unsupported attribute type."} shodan_parser = ShodanParser(attribute) - shodan_parser.query_shodan(request['config']['apikey']) + shodan_parser.query_shodan(request["config"]["apikey"]) return shodan_parser.get_result() @@ -240,5 +237,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/sigma_queries.py b/misp_modules/modules/expansion/sigma_queries.py index 41ba9b49e..452080149 100644 --- a/misp_modules/modules/expansion/sigma_queries.py +++ b/misp_modules/modules/expansion/sigma_queries.py @@ -1,45 +1,59 @@ import io import json -try: - from sigma.parser.collection import SigmaCollectionParser - from sigma.configuration import SigmaConfiguration - from sigma.backends.discovery import getBackend -except ImportError: - print("sigma or yaml is missing, use 'pip3 install sigmatools' to install it.") - -misperrors = {'error': 'Error'} -mispattributes = {'input': ['sigma'], 'output': ['text']} + +from sigma.backends.discovery import getBackend +from sigma.configuration import SigmaConfiguration +from sigma.parser.collection import SigmaCollectionParser + +misperrors = {"error": "Error"} +mispattributes = {"input": ["sigma"], "output": ["text"]} moduleinfo = { - 'version': '0.1', - 'author': 'Christian Studer', - 'module-type': ['expansion', 'hover'], - 'name': 'Sigma Rule Converter', - 'description': 'An expansion hover module to display the result of sigma queries.', - 'logo': 'sigma.png', - 'requirements': ['Sigma python library'], - 'features': 'This module takes a Sigma rule attribute as input and tries all the different queries available to convert it into different formats recognized by SIEMs.', - 'references': ['https://github.com/Neo23x0/sigma/wiki'], - 'input': 'A Sigma attribute.', - 'output': 'Text displaying results of queries on the Sigma attribute.', + "version": "0.1", + "author": "Christian Studer", + "module-type": ["expansion", "hover"], + "name": "Sigma Rule Converter", + "description": "An expansion hover module to display the result of sigma queries.", + "logo": "sigma.png", + "requirements": ["Sigma python library"], + "features": ( + "This module takes a Sigma rule attribute as input and tries all the different queries available to convert it" + " into different formats recognized by SIEMs." + ), + "references": ["https://github.com/Neo23x0/sigma/wiki"], + "input": "A Sigma attribute.", + "output": "Text displaying results of queries on the Sigma attribute.", } moduleconfig = [] -sigma_targets = ('es-dsl', 'es-qs', 'graylog', 'kibana', 'xpack-watcher', 'logpoint', 'splunk', 'grep', 'mdatp', 'splunkxml', 'arcsight', 'qualys') +sigma_targets = ( + "es-dsl", + "es-qs", + "graylog", + "kibana", + "xpack-watcher", + "logpoint", + "splunk", + "grep", + "mdatp", + "splunkxml", + "arcsight", + "qualys", +) def handler(q=False): if q is False: return False request = json.loads(q) - if not request.get('sigma'): - misperrors['error'] = 'Sigma rule missing' + if not request.get("sigma"): + misperrors["error"] = "Sigma rule missing" return misperrors config = SigmaConfiguration() - f = io.TextIOWrapper(io.BytesIO(request.get('sigma').encode()), encoding='utf-8') + f = io.TextIOWrapper(io.BytesIO(request.get("sigma").encode()), encoding="utf-8") parser = SigmaCollectionParser(f, config) targets = [] results = [] for t in sigma_targets: - backend = getBackend(t)(config, {'rulecomment': False}) + backend = getBackend(t)(config, {"rulecomment": False}) try: parser.generate(backend) result = backend.finalize() @@ -49,7 +63,7 @@ def handler(q=False): except Exception: continue d_result = {t: r.strip() for t, r in zip(targets, results)} - return {'results': [{'types': mispattributes['output'], 'values': d_result}]} + return {"results": [{"types": mispattributes["output"], "values": d_result}]} def introspection(): @@ -57,5 +71,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/sigma_syntax_validator.py b/misp_modules/modules/expansion/sigma_syntax_validator.py index b87392332..ccb473ace 100644 --- a/misp_modules/modules/expansion/sigma_syntax_validator.py +++ b/misp_modules/modules/expansion/sigma_syntax_validator.py @@ -1,25 +1,26 @@ import json -try: - import yaml - from sigma.parser.rule import SigmaParser - from sigma.configuration import SigmaConfiguration -except ImportError: - print("sigma or yaml is missing, use 'pip3 install sigmatools' to install it.") - -misperrors = {'error': 'Error'} -mispattributes = {'input': ['sigma'], 'output': ['text']} + +import yaml +from sigma.configuration import SigmaConfiguration +from sigma.parser.rule import SigmaParser + +misperrors = {"error": "Error"} +mispattributes = {"input": ["sigma"], "output": ["text"]} moduleinfo = { - 'version': '0.1', - 'author': 'Christian Studer', - 'module-type': ['expansion', 'hover'], - 'name': 'Sigma Syntax Validator', - 'description': 'An expansion hover module to perform a syntax check on sigma rules.', - 'logo': 'sigma.png', - 'requirements': ['Sigma python library', 'Yaml python library'], - 'features': 'This module takes a Sigma rule attribute as input and performs a syntax check on it.\n\nIt displays then that the rule is valid if it is the case, and the error related to the rule otherwise.', - 'references': ['https://github.com/Neo23x0/sigma/wiki'], - 'input': 'A Sigma attribute.', - 'output': 'Text describing the validity of the Sigma rule.', + "version": "0.1", + "author": "Christian Studer", + "module-type": ["expansion", "hover"], + "name": "Sigma Syntax Validator", + "description": "An expansion hover module to perform a syntax check on sigma rules.", + "logo": "sigma.png", + "requirements": ["Sigma python library", "Yaml python library"], + "features": ( + "This module takes a Sigma rule attribute as input and performs a syntax check on it.\n\nIt displays then that" + " the rule is valid if it is the case, and the error related to the rule otherwise." + ), + "references": ["https://github.com/Neo23x0/sigma/wiki"], + "input": "A Sigma attribute.", + "output": "Text describing the validity of the Sigma rule.", } moduleconfig = [] @@ -28,16 +29,16 @@ def handler(q=False): if q is False: return False request = json.loads(q) - if not request.get('sigma'): - misperrors['error'] = 'Sigma rule missing' + if not request.get("sigma"): + misperrors["error"] = "Sigma rule missing" return misperrors config = SigmaConfiguration() try: - parser = SigmaParser(yaml.safe_load(request.get('sigma')), config) - result = ("Syntax valid: {}".format(parser.values)) + parser = SigmaParser(yaml.safe_load(request.get("sigma")), config) + result = "Syntax valid: {}".format(parser.values) except Exception as e: - result = ("Syntax error: {}".format(str(e))) - return {'results': [{'types': mispattributes['output'], 'values': result}]} + result = "Syntax error: {}".format(str(e)) + return {"results": [{"types": mispattributes["output"], "values": result}]} def introspection(): @@ -45,5 +46,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/sigmf_expand.py b/misp_modules/modules/expansion/sigmf_expand.py index b7a55a807..94fce9c3f 100644 --- a/misp_modules/modules/expansion/sigmf_expand.py +++ b/misp_modules/modules/expansion/sigmf_expand.py @@ -1,43 +1,48 @@ # -*- coding: utf-8 -*- import base64 -import numpy as np -import matplotlib.pyplot as plt import io import json -import tempfile import logging import sys -from pymisp import MISPObject, MISPEvent +import tarfile +import tempfile + +import matplotlib.pyplot as plt +import numpy as np +from pymisp import MISPEvent, MISPObject from sigmf import SigMFFile from sigmf.archive import SIGMF_DATASET_EXT, SIGMF_METADATA_EXT -import tarfile log = logging.getLogger("sigmf-expand") log.setLevel(logging.DEBUG) sh = logging.StreamHandler(sys.stdout) sh.setLevel(logging.DEBUG) -fmt = logging.Formatter( - "%(asctime)s - %(name)s - %(levelname)s - %(message)s" -) +fmt = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") sh.setFormatter(fmt) log.addHandler(sh) -misperrors = {'error': 'Error'} -mispattributes = {'input': ['sigmf-recording', 'sigmf-archive'], 'output': [ - 'MISP objects'], 'format': 'misp_standard'} +misperrors = {"error": "Error"} +mispattributes = { + "input": ["sigmf-recording", "sigmf-archive"], + "output": ["MISP objects"], + "format": "misp_standard", +} moduleinfo = { - 'version': '0.1', - 'author': 'Luciano Righetti', - 'description': 'Expands a SigMF Recording object into a SigMF Expanded Recording object, extracts a SigMF archive into a SigMF Recording object.', - 'module-type': ['expansion'], - 'name': 'SigMF Expansion', - 'logo': '', - 'requirements': [], - 'features': '', - 'references': [], - 'input': '', - 'output': '', + "version": "0.1", + "author": "Luciano Righetti", + "description": ( + "Expands a SigMF Recording object into a SigMF Expanded Recording object, extracts a SigMF archive into a SigMF" + " Recording object." + ), + "module-type": ["expansion"], + "name": "SigMF Expansion", + "logo": "", + "requirements": [], + "features": "", + "references": [], + "input": "", + "output": "", } @@ -75,10 +80,9 @@ def get_samples(data_bytes, data_type) -> np.ndarray: def generate_plots(recording, meta_filename, data_bytes): # FFT plot - filename = meta_filename.replace('.sigmf-data', '') - samples = get_samples( - data_bytes, recording.get_global_info()['core:datatype']) - sample_rate = recording.get_global_info()['core:sample_rate'] + filename = meta_filename.replace(".sigmf-data", "") + samples = get_samples(data_bytes, recording.get_global_info()["core:datatype"]) + sample_rate = recording.get_global_info()["core:sample_rate"] # Waterfall plot # snippet from https://pysdr.org/content/frequency_domain.html#fast-fourier-transform-fft @@ -87,30 +91,38 @@ def generate_plots(recording, meta_filename, data_bytes): num_rows = len(samples) // fft_size spectrogram = np.zeros((num_rows, fft_size)) for i in range(num_rows): - spectrogram[i, :] = 10 * \ - np.log10(np.abs(np.fft.fftshift( - np.fft.fft(samples[i*fft_size:(i+1)*fft_size])))**2) + spectrogram[i, :] = 10 * np.log10( + np.abs(np.fft.fftshift(np.fft.fft(samples[i * fft_size : (i + 1) * fft_size]))) ** 2 + ) plt.figure(figsize=(10, 4)) plt.title(filename) - plt.imshow(spectrogram, aspect='auto', extent=[ - sample_rate/-2/1e6, sample_rate/2/1e6, 0, len(samples)/sample_rate]) + plt.imshow( + spectrogram, + aspect="auto", + extent=[ + sample_rate / -2 / 1e6, + sample_rate / 2 / 1e6, + 0, + len(samples) / sample_rate, + ], + ) plt.xlabel("Frequency [MHz]") plt.ylabel("Time [ms]") - plt.savefig(filename + '-spectrogram.png') + plt.savefig(filename + "-spectrogram.png") waterfall_buff = io.BytesIO() - plt.savefig(waterfall_buff, format='png') + plt.savefig(waterfall_buff, format="png") waterfall_buff.seek(0) - waterfall_png = base64.b64encode(waterfall_buff.read()).decode('utf-8') + waterfall_png = base64.b64encode(waterfall_buff.read()).decode("utf-8") waterfall_attr = { - 'type': 'attachment', - 'value': filename + '-waterfall.png', - 'data': waterfall_png, - 'comment': 'Waterfall plot of the recording' + "type": "attachment", + "value": filename + "-waterfall.png", + "data": waterfall_png, + "comment": "Waterfall plot of the recording", } - return [{'relation': 'waterfall-plot', 'attribute': waterfall_attr}] + return [{"relation": "waterfall-plot", "attribute": waterfall_attr}] def process_sigmf_archive(object): @@ -121,19 +133,17 @@ def process_sigmf_archive(object): try: # get sigmf-archive attribute - for attribute in object['Attribute']: - if attribute['object_relation'] == 'SigMF-archive': + for attribute in object["Attribute"]: + if attribute["object_relation"] == "SigMF-archive": # write temp data file to disk - sigmf_archive_file = tempfile.NamedTemporaryFile( - suffix='.sigmf') - sigmf_archive_bin = base64.b64decode(attribute['data']) - with open(sigmf_archive_file.name, 'wb') as f: + sigmf_archive_file = tempfile.NamedTemporaryFile(suffix=".sigmf") + sigmf_archive_bin = base64.b64decode(attribute["data"]) + with open(sigmf_archive_file.name, "wb") as f: f.write(sigmf_archive_bin) f.close() - sigmf_tarfile = tarfile.open( - sigmf_archive_file.name, mode="r", format=tarfile.PAX_FORMAT) + sigmf_tarfile = tarfile.open(sigmf_archive_file.name, mode="r", format=tarfile.PAX_FORMAT) files = sigmf_tarfile.getmembers() @@ -141,37 +151,37 @@ def process_sigmf_archive(object): if file.name.endswith(SIGMF_METADATA_EXT): metadata_reader = sigmf_tarfile.extractfile(file) sigmf_meta_attr = { - 'type': 'attachment', - 'value': file.name, - 'data': base64.b64encode(metadata_reader.read()).decode("utf-8"), - 'comment': 'SigMF metadata file', - 'object_relation': 'SigMF-meta' + "type": "attachment", + "value": file.name, + "data": base64.b64encode(metadata_reader.read()).decode("utf-8"), + "comment": "SigMF metadata file", + "object_relation": "SigMF-meta", } if file.name.endswith(SIGMF_DATASET_EXT): data_reader = sigmf_tarfile.extractfile(file) sigmf_data_attr = { - 'type': 'attachment', - 'value': file.name, - 'data': base64.b64encode(data_reader.read()).decode("utf-8"), - 'comment': 'SigMF data file', - 'object_relation': 'SigMF-data' + "type": "attachment", + "value": file.name, + "data": base64.b64encode(data_reader.read()).decode("utf-8"), + "comment": "SigMF data file", + "object_relation": "SigMF-data", } if sigmf_meta_attr is None: return {"error": "No SigMF metadata file found"} - recording = MISPObject('sigmf-recording') + recording = MISPObject("sigmf-recording") recording.add_attribute(**sigmf_meta_attr) recording.add_attribute(**sigmf_data_attr) # add reference to original SigMF Archive object - recording.add_reference(object['uuid'], "expands") + recording.add_reference(object["uuid"], "expands") event.add_object(recording) event = json.loads(event.to_json()) - return {"results": {'Object': event['Object']}} + return {"results": {"Object": event["Object"]}} # no sigmf-archive attribute found return {"error": "No SigMF-archive attribute found"} @@ -185,11 +195,11 @@ def process_sigmf_recording(object): event = MISPEvent() - for attribute in object['Attribute']: - if attribute['object_relation'] == 'SigMF-data': + for attribute in object["Attribute"]: + if attribute["object_relation"] == "SigMF-data": sigmf_data_attr = attribute - if attribute['object_relation'] == 'SigMF-meta': + if attribute["object_relation"] == "SigMF-meta": sigmf_meta_attr = attribute if sigmf_meta_attr is None: @@ -199,95 +209,96 @@ def process_sigmf_recording(object): return {"error": "No SigMF-meta attribute"} try: - sigmf_meta = base64.b64decode(sigmf_meta_attr['data']).decode('utf-8') + sigmf_meta = base64.b64decode(sigmf_meta_attr["data"]).decode("utf-8") sigmf_meta = json.loads(sigmf_meta) except Exception as e: logging.exception(e) return {"error": "Provided .sigmf-meta is not a valid JSON string"} # write temp data file to disk - sigmf_data_file = tempfile.NamedTemporaryFile(suffix='.sigmf-data') - sigmf_data_bin = base64.b64decode(sigmf_data_attr['data']) - with open(sigmf_data_file.name, 'wb') as f: + sigmf_data_file = tempfile.NamedTemporaryFile(suffix=".sigmf-data") + sigmf_data_bin = base64.b64decode(sigmf_data_attr["data"]) + with open(sigmf_data_file.name, "wb") as f: f.write(sigmf_data_bin) f.close() try: - recording = SigMFFile( - metadata=sigmf_meta, - data_file=sigmf_data_file.name - ) + recording = SigMFFile(metadata=sigmf_meta, data_file=sigmf_data_file.name) except Exception as e: logging.exception(e) return {"error": "Provided .sigmf-meta and .sigmf-data is not a valid SigMF file"} - expanded_sigmf = MISPObject('sigmf-expanded-recording') + expanded_sigmf = MISPObject("sigmf-expanded-recording") - if 'core:author' in sigmf_meta['global']: - expanded_sigmf.add_attribute( - 'author', **{'type': 'text', 'value': sigmf_meta['global']['core:author']}) - if 'core:datatype' in sigmf_meta['global']: - expanded_sigmf.add_attribute( - 'datatype', **{'type': 'text', 'value': sigmf_meta['global']['core:datatype']}) - if 'core:description' in sigmf_meta['global']: + if "core:author" in sigmf_meta["global"]: + expanded_sigmf.add_attribute("author", **{"type": "text", "value": sigmf_meta["global"]["core:author"]}) + if "core:datatype" in sigmf_meta["global"]: expanded_sigmf.add_attribute( - 'description', **{'type': 'text', 'value': sigmf_meta['global']['core:description']}) - if 'core:license' in sigmf_meta['global']: - expanded_sigmf.add_attribute( - 'license', **{'type': 'text', 'value': sigmf_meta['global']['core:license']}) - if 'core:num_channels' in sigmf_meta['global']: - expanded_sigmf.add_attribute( - 'num_channels', **{'type': 'counter', 'value': sigmf_meta['global']['core:num_channels']}) - if 'core:recorder' in sigmf_meta['global']: + "datatype", + **{"type": "text", "value": sigmf_meta["global"]["core:datatype"]}, + ) + if "core:description" in sigmf_meta["global"]: expanded_sigmf.add_attribute( - 'recorder', **{'type': 'text', 'value': sigmf_meta['global']['core:recorder']}) - if 'core:sample_rate' in sigmf_meta['global']: + "description", + **{"type": "text", "value": sigmf_meta["global"]["core:description"]}, + ) + if "core:license" in sigmf_meta["global"]: + expanded_sigmf.add_attribute("license", **{"type": "text", "value": sigmf_meta["global"]["core:license"]}) + if "core:num_channels" in sigmf_meta["global"]: expanded_sigmf.add_attribute( - 'sample_rate', **{'type': 'float', 'value': sigmf_meta['global']['core:sample_rate']}) - if 'core:sha512' in sigmf_meta['global']: + "num_channels", + **{"type": "counter", "value": sigmf_meta["global"]["core:num_channels"]}, + ) + if "core:recorder" in sigmf_meta["global"]: expanded_sigmf.add_attribute( - 'sha512', **{'type': 'text', 'value': sigmf_meta['global']['core:sha512']}) - if 'core:version' in sigmf_meta['global']: + "recorder", + **{"type": "text", "value": sigmf_meta["global"]["core:recorder"]}, + ) + if "core:sample_rate" in sigmf_meta["global"]: expanded_sigmf.add_attribute( - 'version', **{'type': 'text', 'value': sigmf_meta['global']['core:version']}) + "sample_rate", + **{"type": "float", "value": sigmf_meta["global"]["core:sample_rate"]}, + ) + if "core:sha512" in sigmf_meta["global"]: + expanded_sigmf.add_attribute("sha512", **{"type": "text", "value": sigmf_meta["global"]["core:sha512"]}) + if "core:version" in sigmf_meta["global"]: + expanded_sigmf.add_attribute("version", **{"type": "text", "value": sigmf_meta["global"]["core:version"]}) # add reference to original SigMF Recording object - expanded_sigmf.add_reference(object['uuid'], "expands") + expanded_sigmf.add_reference(object["uuid"], "expands") # add FFT and waterfall plot try: - plots = generate_plots( - recording, sigmf_data_attr['value'], sigmf_data_bin) + plots = generate_plots(recording, sigmf_data_attr["value"], sigmf_data_bin) except Exception as e: logging.exception(e) return {"error": "Could not generate plots"} for plot in plots: - expanded_sigmf.add_attribute(plot['relation'], **plot['attribute']) + expanded_sigmf.add_attribute(plot["relation"], **plot["attribute"]) event.add_object(expanded_sigmf) event = json.loads(event.to_json()) - return {"results": {'Object': event['Object']}} + return {"results": {"Object": event["Object"]}} def handler(q=False): request = json.loads(q) object = request.get("object") - event = MISPEvent() if not object: return {"error": "No object provided"} - if 'Attribute' not in object: + if "Attribute" not in object: return {"error": "Empty Attribute list"} # check if it's a SigMF Archive - if object['name'] == 'sigmf-archive': + if object["name"] == "sigmf-archive": return process_sigmf_archive(object) # check if it's a SigMF Recording - if object['name'] == 'sigmf-recording': + if object["name"] == "sigmf-recording": return process_sigmf_recording(object) # TODO: add support for SigMF Collection diff --git a/misp_modules/modules/expansion/socialscan.py b/misp_modules/modules/expansion/socialscan.py index 8800397b4..993814e5b 100644 --- a/misp_modules/modules/expansion/socialscan.py +++ b/misp_modules/modules/expansion/socialscan.py @@ -1,31 +1,41 @@ import json + from socialscan.platforms import Platforms from socialscan.util import sync_execute_queries moduleinfo = { - 'version': '1', - 'author': 'Christian Studer', - 'description': 'A hover module to get information on the availability of an email address or username on some online platforms.', - 'module-type': ['hover'], - 'name': 'Socialscan Lookup', - 'logo': '', - 'requirements': ['The socialscan python library'], - 'features': 'The module takes an email address or username as input and check its availability on some online platforms. The results for each platform are then returned to see if the email address or the username is used, available or if there is an issue with it.', - 'references': ['https://github.com/iojw/socialscan'], - 'input': 'An email address or usename attribute.', - 'output': 'Text containing information about the availability of an email address or a username in some online platforms.', + "version": "1", + "author": "Christian Studer", + "description": ( + "A hover module to get information on the availability of an email address or username on some online" + " platforms." + ), + "module-type": ["hover"], + "name": "Socialscan Lookup", + "logo": "", + "requirements": ["The socialscan python library"], + "features": ( + "The module takes an email address or username as input and check its availability on some online platforms." + " The results for each platform are then returned to see if the email address or the username is used," + " available or if there is an issue with it." + ), + "references": ["https://github.com/iojw/socialscan"], + "input": "An email address or usename attribute.", + "output": ( + "Text containing information about the availability of an email address or a username in some online platforms." + ), } mispattributes = { - 'input': [ - 'github-username', - 'target-user', - 'email', - 'email-src', - 'email-dst', - 'target-email', - 'whois-registrant-email' + "input": [ + "github-username", + "target-user", + "email", + "email-src", + "email-dst", + "target-email", + "whois-registrant-email", ], - 'output': ['text'] + "output": ["text"], } moduleconfig = [] @@ -34,19 +44,15 @@ Platforms.TWITTER, Platforms.GITHUB, Platforms.TUMBLR, - Platforms.LASTFM -] -_EMAIL_PLATFORMS = [ - Platforms.PINTEREST, - Platforms.SPOTIFY, - Platforms.FIREFOX + Platforms.LASTFM, ] +_EMAIL_PLATFORMS = [Platforms.PINTEREST, Platforms.SPOTIFY, Platforms.FIREFOX] _EMAIL_PLATFORMS.extend(_PLATFORMS) _USERNAME_PLATFORMS = [ Platforms.SNAPCHAT, Platforms.GITLAB, Platforms.REDDIT, - Platforms.YAHOO + Platforms.YAHOO, ] _USERNAME_PLATFORMS.extend(_PLATFORMS) @@ -55,30 +61,25 @@ def parse_results(query_results, feature): results = [] for result in query_results: if not result.success: - results.append(f'Unable to retrieve the {feature} on {result.platform}.') + results.append(f"Unable to retrieve the {feature} on {result.platform}.") continue if not result.valid: - results.append(f'Invalid response from {result.platform}, or invalid {feature}.') + results.append(f"Invalid response from {result.platform}, or invalid {feature}.") continue - statement = 'No account' if result.available else 'There is an account' - results.append(f'{statement} linked to the {feature} on {result.platform}.') - to_return = [ - { - 'types': mispattributes['output'], - 'values': result - } for result in results - ] - return {'results': to_return} + statement = "No account" if result.available else "There is an account" + results.append(f"{statement} linked to the {feature} on {result.platform}.") + to_return = [{"types": mispattributes["output"], "values": result} for result in results] + return {"results": to_return} def parse_email(email): results = sync_execute_queries([email], platforms=_EMAIL_PLATFORMS) - return parse_results(results, 'email address') + return parse_results(results, "email address") def parse_username(username, platforms=_USERNAME_PLATFORMS): results = sync_execute_queries([username], platforms=platforms) - return parse_results(results, 'username') + return parse_results(results, "username") def parse_github_username(username): @@ -89,14 +90,14 @@ def handler(q=False): if q is False: return False request = json.loads(q) - if request.get('github-username'): - return parse_github_username(request['github-username']) - if request.get('target-user'): - return parse_username(request['target-user']) - for attribute_type in mispattributes['input'][2:]: + if request.get("github-username"): + return parse_github_username(request["github-username"]) + if request.get("target-user"): + return parse_username(request["target-user"]) + for attribute_type in mispattributes["input"][2:]: if request.get(attribute_type): return parse_email(request[attribute_type]) - return {'error': 'Unsupported attributes type'} + return {"error": "Unsupported attributes type"} def introspection(): @@ -104,5 +105,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/sophoslabs_intelix.py b/misp_modules/modules/expansion/sophoslabs_intelix.py index 1b0bc9ff8..10d4ff34b 100644 --- a/misp_modules/modules/expansion/sophoslabs_intelix.py +++ b/misp_modules/modules/expansion/sophoslabs_intelix.py @@ -1,109 +1,147 @@ +import base64 import json +from urllib.parse import quote + import requests -import base64 -from . import check_input_attribute, checking_error, standard_error_message from pymisp import MISPEvent, MISPObject -from urllib.parse import quote + +from . import check_input_attribute, checking_error, standard_error_message moduleinfo = { - 'version': '1.0', - 'author': 'Ben Verschaeren', - 'description': 'An expansion module to query the Sophoslabs intelix API to get additional information about an ip address, url, domain or sha256 attribute.', - 'module-type': ['expansion'], - 'name': 'SophosLabs Intelix Lookup', - 'logo': 'sophoslabs_intelix.svg', - 'requirements': ['A client_id and client_secret pair to authenticate to the SophosLabs Intelix API'], - 'features': 'The module takes an ip address, url, domain or sha256 attribute and queries the SophosLabs Intelix API with the attribute value. The result of this query is a SophosLabs Intelix hash report, or an ip or url lookup, that is then parsed and returned in a MISP object.', - 'references': ['https://aws.amazon.com/marketplace/pp/B07SLZPMCS'], - 'input': 'An ip address, url, domain or sha256 attribute.', - 'output': 'SophosLabs Intelix report and lookup objects', + "version": "1.0", + "author": "Ben Verschaeren", + "description": ( + "An expansion module to query the Sophoslabs intelix API to get additional information about an ip address," + " url, domain or sha256 attribute." + ), + "module-type": ["expansion"], + "name": "SophosLabs Intelix Lookup", + "logo": "sophoslabs_intelix.svg", + "requirements": ["A client_id and client_secret pair to authenticate to the SophosLabs Intelix API"], + "features": ( + "The module takes an ip address, url, domain or sha256 attribute and queries the SophosLabs Intelix API with" + " the attribute value. The result of this query is a SophosLabs Intelix hash report, or an ip or url lookup," + " that is then parsed and returned in a MISP object." + ), + "references": ["https://aws.amazon.com/marketplace/pp/B07SLZPMCS"], + "input": "An ip address, url, domain or sha256 attribute.", + "output": "SophosLabs Intelix report and lookup objects", } -moduleconfig = ['client_id', 'client_secret'] +moduleconfig = ["client_id", "client_secret"] -misperrors = {'error': 'Error'} +misperrors = {"error": "Error"} -misp_types_in = ['sha256', 'ip', 'ip-src', 'ip-dst', 'uri', 'url', 'domain', 'hostname'] +misp_types_in = ["sha256", "ip", "ip-src", "ip-dst", "uri", "url", "domain", "hostname"] -mispattributes = {'input': misp_types_in, - 'format': 'misp_standard'} +mispattributes = {"input": misp_types_in, "format": "misp_standard"} -class SophosLabsApi(): +class SophosLabsApi: def __init__(self, client_id, client_secret): self.misp_event = MISPEvent() self.client_id = client_id self.client_secret = client_secret self.authToken = f"{self.client_id}:{self.client_secret}" - self.baseurl = 'de.api.labs.sophos.com' - d = {'grant_type': 'client_credentials'} - h = {'Authorization': f"Basic {base64.b64encode(self.authToken.encode('UTF-8')).decode('ascii')}", - 'Content-Type': 'application/x-www-form-urlencoded'} - r = requests.post('https://api.labs.sophos.com/oauth2/token', headers=h, data=d) + self.baseurl = "de.api.labs.sophos.com" + d = {"grant_type": "client_credentials"} + h = { + "Authorization": f"Basic {base64.b64encode(self.authToken.encode('UTF-8')).decode('ascii')}", + "Content-Type": "application/x-www-form-urlencoded", + } + r = requests.post("https://api.labs.sophos.com/oauth2/token", headers=h, data=d) if r.status_code == 200: j = json.loads(r.text) - self.accessToken = j['access_token'] + self.accessToken = j["access_token"] def get_result(self): event = json.loads(self.misp_event.to_json()) - results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} - return {'results': results} + results = {key: event[key] for key in ("Attribute", "Object") if (key in event and event[key])} + return {"results": results} def hash_lookup(self, filehash): - sophos_object = MISPObject('SOPHOSLabs Intelix SHA256 Report') + sophos_object = MISPObject("SOPHOSLabs Intelix SHA256 Report") h = {"Authorization": f"{self.accessToken}"} r = requests.get(f"https://{self.baseurl}/lookup/files/v1/{filehash}", headers=h) if r.status_code == 200: j = json.loads(r.text) - if 'reputationScore' in j: - sophos_object.add_attribute('Reputation Score', type='text', value=j['reputationScore']) - if 0 <= j['reputationScore'] <= 19: - sophos_object.add_attribute('Decision', type='text', value='This file is malicious') - if 20 <= j['reputationScore'] <= 29: - sophos_object.add_attribute('Decision', type='text', value='This file is potentially unwanted') - if 30 <= j['reputationScore'] <= 69: - sophos_object.add_attribute('Decision', type='text', value='This file is unknown and suspicious') - if 70 <= j['reputationScore'] <= 100: - sophos_object.add_attribute('Decision', type='text', value='This file is known good') - if 'detectionName' in j: - sophos_object.add_attribute('Detection Name', type='text', value=j['detectionName']) + if "reputationScore" in j: + sophos_object.add_attribute("Reputation Score", type="text", value=j["reputationScore"]) + if 0 <= j["reputationScore"] <= 19: + sophos_object.add_attribute("Decision", type="text", value="This file is malicious") + if 20 <= j["reputationScore"] <= 29: + sophos_object.add_attribute( + "Decision", + type="text", + value="This file is potentially unwanted", + ) + if 30 <= j["reputationScore"] <= 69: + sophos_object.add_attribute( + "Decision", + type="text", + value="This file is unknown and suspicious", + ) + if 70 <= j["reputationScore"] <= 100: + sophos_object.add_attribute("Decision", type="text", value="This file is known good") + if "detectionName" in j: + sophos_object.add_attribute("Detection Name", type="text", value=j["detectionName"]) else: - sophos_object.add_attribute('Detection Name', type='text', value='No name associated with this IoC') + sophos_object.add_attribute( + "Detection Name", + type="text", + value="No name associated with this IoC", + ) self.misp_event.add_object(**sophos_object) def ip_lookup(self, ip): - sophos_object = MISPObject('SOPHOSLabs Intelix IP Category Lookup') + sophos_object = MISPObject("SOPHOSLabs Intelix IP Category Lookup") h = {"Authorization": f"{self.accessToken}"} r = requests.get(f"https://{self.baseurl}/lookup/ips/v1/{ip}", headers=h) if r.status_code == 200: j = json.loads(r.text) - if 'category' in j: - for c in j['category']: - sophos_object.add_attribute('IP Address Categorisation', type='text', value=c) + if "category" in j: + for c in j["category"]: + sophos_object.add_attribute("IP Address Categorisation", type="text", value=c) else: - sophos_object.add_attribute('IP Address Categorisation', type='text', value='No category assocaited with IoC') + sophos_object.add_attribute( + "IP Address Categorisation", + type="text", + value="No category assocaited with IoC", + ) self.misp_event.add_object(**sophos_object) def url_lookup(self, url): - sophos_object = MISPObject('SOPHOSLabs Intelix URL Lookup') + sophos_object = MISPObject("SOPHOSLabs Intelix URL Lookup") h = {"Authorization": f"{self.accessToken}"} r = requests.get(f"https://{self.baseurl}/lookup/urls/v1/{quote(url, safe='')}", headers=h) if r.status_code == 200: j = json.loads(r.text) - if 'productivityCategory' in j: - sophos_object.add_attribute('URL Categorisation', type='text', value=j['productivityCategory']) + if "productivityCategory" in j: + sophos_object.add_attribute("URL Categorisation", type="text", value=j["productivityCategory"]) else: - sophos_object.add_attribute('URL Categorisation', type='text', value='No category assocaited with IoC') - - if 'riskLevel' in j: - sophos_object.add_attribute('URL Risk Level', type='text', value=j['riskLevel']) + sophos_object.add_attribute( + "URL Categorisation", + type="text", + value="No category assocaited with IoC", + ) + + if "riskLevel" in j: + sophos_object.add_attribute("URL Risk Level", type="text", value=j["riskLevel"]) else: - sophos_object.add_attribute('URL Risk Level', type='text', value='No risk level associated with IoC') - - if 'securityCategory' in j: - sophos_object.add_attribute('URL Security Category', type='text', value=j['securityCategory']) + sophos_object.add_attribute( + "URL Risk Level", + type="text", + value="No risk level associated with IoC", + ) + + if "securityCategory" in j: + sophos_object.add_attribute("URL Security Category", type="text", value=j["securityCategory"]) else: - sophos_object.add_attribute('URL Security Category', type='text', value='No Security Category associated with IoC') + sophos_object.add_attribute( + "URL Security Category", + type="text", + value="No Security Category associated with IoC", + ) self.misp_event.add_object(**sophos_object) @@ -111,29 +149,33 @@ def handler(q=False): if q is False: return False j = json.loads(q) - if not j.get('config') or not j['config'].get('client_id') or not j['config'].get('client_secret'): - misperrors['error'] = "Missing client_id or client_secret value for SOPHOSLabs Intelix. \ - It's free to sign up here https://aws.amazon.com/marketplace/pp/B07SLZPMCS." + if not j.get("config") or not j["config"].get("client_id") or not j["config"].get("client_secret"): + misperrors["error"] = ( + "Missing client_id or client_secret value for SOPHOSLabs Intelix. It's free to sign up here" + " https://aws.amazon.com/marketplace/pp/B07SLZPMCS." + ) return misperrors - to_check = (('type', 'value'), ('type', 'value1')) - if not j.get('attribute') or not any(check_input_attribute(j['attribute'], requirements=check) for check in to_check): - return {'error': f'{standard_error_message}, {checking_error}.'} - attribute = j['attribute'] - if attribute['type'] not in misp_types_in: - return {'error': 'Unsupported attribute type.'} - client = SophosLabsApi(j['config']['client_id'], j['config']['client_secret']) + to_check = (("type", "value"), ("type", "value1")) + if not j.get("attribute") or not any( + check_input_attribute(j["attribute"], requirements=check) for check in to_check + ): + return {"error": f"{standard_error_message}, {checking_error}."} + attribute = j["attribute"] + if attribute["type"] not in misp_types_in: + return {"error": "Unsupported attribute type."} + client = SophosLabsApi(j["config"]["client_id"], j["config"]["client_secret"]) mapping = { - 'sha256': 'hash_lookup', - 'ip-dst': 'ip_lookup', - 'ip-src': 'ip_lookup', - 'ip': 'ip_lookup', - 'uri': 'url_lookup', - 'url': 'url_lookup', - 'domain': 'url_lookup', - 'hostname': 'url_lookup' + "sha256": "hash_lookup", + "ip-dst": "ip_lookup", + "ip-src": "ip_lookup", + "ip": "ip_lookup", + "uri": "url_lookup", + "url": "url_lookup", + "domain": "url_lookup", + "hostname": "url_lookup", } - attribute_value = attribute['value'] if 'value' in attribute else attribute['value1'] - getattr(client, mapping[attribute['type']])(attribute_value) + attribute_value = attribute["value"] if "value" in attribute else attribute["value1"] + getattr(client, mapping[attribute["type"]])(attribute_value) return client.get_result() @@ -142,5 +184,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/sourcecache.py b/misp_modules/modules/expansion/sourcecache.py index 18b38e422..1e6047de0 100755 --- a/misp_modules/modules/expansion/sourcecache.py +++ b/misp_modules/modules/expansion/sourcecache.py @@ -1,45 +1,51 @@ import json + from url_archiver import url_archiver -misperrors = {'error': 'Error'} -mispattributes = {'input': ['link', 'url'], 'output': ['attachment', 'malware-sample']} +misperrors = {"error": "Error"} +mispattributes = {"input": ["link", "url"], "output": ["attachment", "malware-sample"]} moduleinfo = { - 'version': '0.1', - 'author': 'Alexandre Dulaunoy', - 'description': 'Module to cache web pages of analysis reports, OSINT sources. The module returns a link of the cached page.', - 'module-type': ['expansion'], - 'name': 'URL Archiver', - 'logo': '', - 'requirements': ['urlarchiver: python library to fetch and archive URL on the file-system'], - 'features': 'This module takes a link or url attribute as input and caches the related web page. It returns then a link of the cached page.', - 'references': ['https://github.com/adulau/url_archiver'], - 'input': 'A link or url attribute.', - 'output': 'A malware-sample attribute describing the cached page.', + "version": "0.1", + "author": "Alexandre Dulaunoy", + "description": ( + "Module to cache web pages of analysis reports, OSINT sources. The module returns a link of the cached page." + ), + "module-type": ["expansion"], + "name": "URL Archiver", + "logo": "", + "requirements": ["urlarchiver: python library to fetch and archive URL on the file-system"], + "features": ( + "This module takes a link or url attribute as input and caches the related web page. It returns then a link of" + " the cached page." + ), + "references": ["https://github.com/adulau/url_archiver"], + "input": "A link or url attribute.", + "output": "A malware-sample attribute describing the cached page.", } -moduleconfig = ['archivepath'] +moduleconfig = ["archivepath"] def handler(q=False): if q is False: return False request = json.loads(q) - if (request.get('config')): - archive_path = request['config']['archivepath'] + if request.get("config"): + archive_path = request["config"]["archivepath"] else: - archive_path = '/tmp/' - if request.get('link'): - tocache = request['link'] + archive_path = "/tmp/" + if request.get("link"): + tocache = request["link"] data = __archiveLink(archive_path, tocache) - mispattributes['output'] = ['attachment'] - elif request.get('url'): - tocache = request['url'] + mispattributes["output"] = ["attachment"] + elif request.get("url"): + tocache = request["url"] data = __archiveLink(archive_path, tocache) - mispattributes['output'] = ['malware-sample'] + mispattributes["output"] = ["malware-sample"] else: - misperrors['error'] = "Link is missing" + misperrors["error"] = "Link is missing" return misperrors - enc_data = data.decode('ascii') - r = {'results': [{'types': mispattributes['output'], 'values': tocache, 'data': enc_data}]} + enc_data = data.decode("ascii") + r = {"results": [{"types": mispattributes["output"], "values": tocache, "data": enc_data}]} return r @@ -53,5 +59,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/stairwell.py b/misp_modules/modules/expansion/stairwell.py index 1421240a4..9140aa12f 100644 --- a/misp_modules/modules/expansion/stairwell.py +++ b/misp_modules/modules/expansion/stairwell.py @@ -1,55 +1,84 @@ import json import re + import requests from pymisp import MISPEvent, MISPObject -from . import check_input_attribute, checking_error, standard_error_message +from . import check_input_attribute, checking_error, standard_error_message -misperrors = { - 'error': 'Error' -} -mispattributes = { - 'input': [ - 'md5', - 'sha1', - 'sha256' - ], - 'format': 'misp_standard' -} +misperrors = {"error": "Error"} +mispattributes = {"input": ["md5", "sha1", "sha256"], "format": "misp_standard"} moduleinfo = { - 'version': '0.1', - 'author': 'goodlandsecurity', - 'description': 'Module to query the Stairwell API to get additional information about the input hash attribute', - 'module-type': ['expansion'], - 'name': 'Stairwell Lookup', - 'logo': 'stairwell.png', - 'requirements': ['Access to Stairwell platform (apikey)'], - 'features': "The module takes a hash attribute as input and queries Stariwell's API to fetch additional data about it. The result, if the payload is observed in Stariwell, is a file object describing the file the input hash is related to.", - 'references': ['https://stairwell.com', 'https://docs.stairwell.com'], - 'input': 'A hash attribute (md5, sha1, sha256).', - 'output': 'File object related to the input attribute found on Stairwell platform.', + "version": "0.1", + "author": "goodlandsecurity", + "description": "Module to query the Stairwell API to get additional information about the input hash attribute", + "module-type": ["expansion"], + "name": "Stairwell Lookup", + "logo": "stairwell.png", + "requirements": ["Access to Stairwell platform (apikey)"], + "features": ( + "The module takes a hash attribute as input and queries Stariwell's API to fetch additional data about it. The" + " result, if the payload is observed in Stariwell, is a file object describing the file the input hash is" + " related to." + ), + "references": ["https://stairwell.com", "https://docs.stairwell.com"], + "input": "A hash attribute (md5, sha1, sha256).", + "output": "File object related to the input attribute found on Stairwell platform.", } moduleconfig = ["apikey"] def parse_response(response: dict): attribute_mapping = { - 'environments': {'type': 'comment', 'object_relation': 'environment', 'distribution': 5}, - 'imphash': {'type': 'imphash', 'object_relation': 'impash', 'distribution': 5}, - 'magic': {'type': 'comment', 'object_relation': 'magic', 'distribution': 5}, - 'malEval': { - 'probabilityBucket': {'type': 'comment', 'object_relation': 'malEval-probability', 'distribution': 5}, - 'severity': {'type': 'comment', 'object_relation': 'malEval-severity', 'distribution': 5} + "environments": { + "type": "comment", + "object_relation": "environment", + "distribution": 5, + }, + "imphash": {"type": "imphash", "object_relation": "impash", "distribution": 5}, + "magic": {"type": "comment", "object_relation": "magic", "distribution": 5}, + "malEval": { + "probabilityBucket": { + "type": "comment", + "object_relation": "malEval-probability", + "distribution": 5, + }, + "severity": { + "type": "comment", + "object_relation": "malEval-severity", + "distribution": 5, + }, + }, + "md5": {"type": "md5", "object_relation": "md5", "distribution": 5}, + "mimeType": { + "type": "mime-type", + "object_relation": "mime-type", + "distribution": 5, + }, + "sha1": {"type": "sha1", "object_relation": "sha1", "distribution": 5}, + "sha256": {"type": "sha256", "object_relation": "sha256", "distribution": 5}, + "shannonEntropy": { + "type": "float", + "object_relation": "entropy", + "distribution": 5, + }, + "size": { + "type": "size-in-bytes", + "object_relation": "size-in-bytes", + "distribution": 5, + }, + "stairwellFirstSeenTime": { + "type": "datetime", + "object_relation": "stairwell-first-seen", + "distribution": 5, + }, + "tlsh": {"type": "tlsh", "object_relation": "tlsh", "distribution": 5}, + "yaraRuleMatches": { + "type": "text", + "object_relation": "yara-rule-match", + "comment": "matching Stairwell yara rule name", + "distribution": 5, }, - 'md5': {'type': 'md5', 'object_relation': 'md5', 'distribution': 5}, - 'mimeType': {'type': 'mime-type', 'object_relation': 'mime-type', 'distribution': 5}, - 'sha1': {'type': 'sha1', 'object_relation': 'sha1', 'distribution': 5}, - 'sha256': {'type': 'sha256', 'object_relation': 'sha256', 'distribution': 5}, - 'shannonEntropy': {'type': 'float', 'object_relation': 'entropy', 'distribution': 5}, - 'size': {'type': 'size-in-bytes', 'object_relation': 'size-in-bytes', 'distribution': 5}, - 'stairwellFirstSeenTime': {'type': 'datetime', 'object_relation': 'stairwell-first-seen', 'distribution': 5}, - 'tlsh': {'type': 'tlsh', 'object_relation': 'tlsh', 'distribution': 5}, - 'yaraRuleMatches': {'type': 'text', 'object_relation': 'yara-rule-match', 'comment': 'matching Stairwell yara rule name', 'distribution': 5} } environments_mapping = { "NCS2SM-YHB2KT-SAFUDX-JC7F6WYA": "Florian's Open Rules", @@ -67,82 +96,82 @@ def parse_response(response: dict): "TT9GM5-JUMD8H-9828FL-GAW5NNXE": "stairwell-public-verdicts", "MKYSAR-3XN9MB-3VAK3R-888ZJUTJ": "Threat Report Feeds", "6HP5R3-ZM7DAN-RB4732-X6QPCJ36": "Virusshare", - "TV6WCV-7Y79LE-BK79EY-C8GUEY46": "vxintel" + "TV6WCV-7Y79LE-BK79EY-C8GUEY46": "vxintel", } misp_event = MISPEvent() - misp_object = MISPObject('stairwell') + misp_object = MISPObject("stairwell") for feature, attribute in attribute_mapping.items(): if feature in response.keys() and response[feature]: - if feature == 'yaraRuleMatches': + if feature == "yaraRuleMatches": for rule in response[feature]: - env_pattern = r'\b[A-Z0-9]{6}-[A-Z0-9]{6}-[A-Z0-9]{6}-[A-Z0-9]{8}\b' - env = re.findall(env_pattern, rule.split('yaraRules/')[0])[0] + env_pattern = r"\b[A-Z0-9]{6}-[A-Z0-9]{6}-[A-Z0-9]{6}-[A-Z0-9]{8}\b" + env = re.findall(env_pattern, rule.split("yaraRules/")[0])[0] misp_attribute = { - 'value': rule.split('yaraRules/')[1], - 'comment': f'Rule from: {environments_mapping.get(env, "Unknown UUID!")}' + "value": rule.split("yaraRules/")[1], + "comment": f'Rule from: {environments_mapping.get(env, "Unknown UUID!")}', } misp_attribute.update(attribute) misp_object.add_attribute(**misp_attribute) - elif feature == 'environments': + elif feature == "environments": for env in response[feature]: misp_attribute = { - 'value': environments_mapping.get(env, f'Unknown Environment: {env}'), - 'comment': 'Hash observed in' + "value": environments_mapping.get(env, f"Unknown Environment: {env}"), + "comment": "Hash observed in", } misp_attribute.update(attribute) misp_object.add_attribute(**misp_attribute) - elif feature == 'malEval': + elif feature == "malEval": for attr in attribute: - misp_attribute = {'value': response[feature][attr]} + misp_attribute = {"value": response[feature][attr]} misp_attribute.update(attribute[attr]) misp_object.add_attribute(**misp_attribute) else: - misp_attribute = {'value': response[feature]} + misp_attribute = {"value": response[feature]} misp_attribute.update(attribute) attr = misp_object.add_attribute(**misp_attribute) - if feature in ('md5', 'sha1', 'sha256'): - for label in response['malEval']['labels']: + if feature in ("md5", "sha1", "sha256"): + for label in response["malEval"]["labels"]: attr.add_tag(label) misp_event.add_object(**misp_object) event = json.loads(misp_event.to_json()) - results = {'Object': event['Object']} + results = {"Object": event["Object"]} - return {'results': results} + return {"results": results} def handler(q=False): if q is False: return False request = json.loads(q) - if not request.get('config') or not request['config'].get('apikey'): - misperrors['error'] = 'A Stairwell api key is required for this module!' + if not request.get("config") or not request["config"].get("apikey"): + misperrors["error"] = "A Stairwell api key is required for this module!" return misperrors - if not request.get('attribute') or not check_input_attribute(request['attribute'], requirements=('type', 'value')): - misperrors['error'] = f'{standard_error_message}, {checking_error}.' + if not request.get("attribute") or not check_input_attribute(request["attribute"], requirements=("type", "value")): + misperrors["error"] = f"{standard_error_message}, {checking_error}." return misperrors - attribute = request['attribute'] - if attribute['type'] not in mispattributes['input']: - misperrors['error'] = 'Unsupported attribute type!' + attribute = request["attribute"] + if attribute["type"] not in mispattributes["input"]: + misperrors["error"] = "Unsupported attribute type!" return misperrors headers = { "Accept": "application/json", - "Authorization": request['config']['apikey'], - "User-Agent": f"misp-module {__file__} {moduleinfo['version']}" + "Authorization": request["config"]["apikey"], + "User-Agent": f"misp-module {__file__} {moduleinfo['version']}", } url = f"https://app.stairwell.com/v1/objects/{attribute['value']}/metadata" response = requests.get(url=url, headers=headers).json() - if response.get('code') == 16: # bad auth - return {'error': f"{response['message']} Is api key valid?"} - elif response.get('code') == 5: # not found - return {'error': f"{attribute['type']}:{attribute['value']} {response['message']}"} - elif response.get('code') == 2: # encoding/hex: invalid byte - return {'error': response['message']} - elif response.get('code'): # catchall for potential unforeseen errors - return {'error': response['message'], 'code': response['code']} + if response.get("code") == 16: # bad auth + return {"error": f"{response['message']} Is api key valid?"} + elif response.get("code") == 5: # not found + return {"error": f"{attribute['type']}:{attribute['value']} {response['message']}"} + elif response.get("code") == 2: # encoding/hex: invalid byte + return {"error": response["message"]} + elif response.get("code"): # catchall for potential unforeseen errors + return {"error": response["message"], "code": response["code"]} else: return parse_response(response) @@ -152,5 +181,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/stix2_pattern_syntax_validator.py b/misp_modules/modules/expansion/stix2_pattern_syntax_validator.py index 15e44ee5c..345ff8331 100644 --- a/misp_modules/modules/expansion/stix2_pattern_syntax_validator.py +++ b/misp_modules/modules/expansion/stix2_pattern_syntax_validator.py @@ -1,23 +1,27 @@ import json -try: - from stix2patterns.validator import run_validator -except ImportError: - print("stix2 patterns python library is missing, use 'pip3 install stix2-patterns' to install it.") -misperrors = {'error': 'Error'} -mispattributes = {'input': ['stix2-pattern'], 'output': ['text']} +from stix2patterns.validator import run_validator + +misperrors = {"error": "Error"} +mispattributes = {"input": ["stix2-pattern"], "output": ["text"]} moduleinfo = { - 'version': '0.1', - 'author': 'Christian Studer', - 'module-type': ['hover'], - 'name': 'STIX2 Pattern Syntax Validator', - 'description': 'An expansion hover module to perform a syntax check on stix2 patterns.', - 'logo': 'stix.png', - 'requirements': ['stix2patterns python library'], - 'features': 'This module takes a STIX2 pattern attribute as input and performs a syntax check on it.\n\nIt displays then that the rule is valid if it is the case, and the error related to the rule otherwise.', - 'references': ['[STIX2.0 patterning specifications](http://docs.oasis-open.org/cti/stix/v2.0/cs01/part5-stix-patterning/stix-v2.0-cs01-part5-stix-patterning.html)'], - 'input': 'A STIX2 pattern attribute.', - 'output': 'Text describing the validity of the STIX2 pattern.', + "version": "0.1", + "author": "Christian Studer", + "module-type": ["hover"], + "name": "STIX2 Pattern Syntax Validator", + "description": "An expansion hover module to perform a syntax check on stix2 patterns.", + "logo": "stix.png", + "requirements": ["stix2patterns python library"], + "features": ( + "This module takes a STIX2 pattern attribute as input and performs a syntax check on it.\n\nIt displays then" + " that the rule is valid if it is the case, and the error related to the rule otherwise." + ), + "references": [ + "[STIX2.0 patterning" + " specifications](http://docs.oasis-open.org/cti/stix/v2.0/cs01/part5-stix-patterning/stix-v2.0-cs01-part5-stix-patterning.html)" + ], + "input": "A STIX2 pattern attribute.", + "output": "Text describing the validity of the STIX2 pattern.", } moduleconfig = [] @@ -26,25 +30,25 @@ def handler(q=False): if q is False: return False request = json.loads(q) - if not request.get('stix2-pattern'): - misperrors['error'] = 'STIX2 pattern missing' + if not request.get("stix2-pattern"): + misperrors["error"] = "STIX2 pattern missing" return misperrors - pattern = request.get('stix2-pattern') + pattern = request.get("stix2-pattern") syntax_errors = [] - for p in pattern[1:-1].split(' AND '): + for p in pattern[1:-1].split(" AND "): syntax_validator = run_validator("[{}]".format(p)) if syntax_validator: for error in syntax_validator: syntax_errors.append(error) if syntax_errors: - s = 's' if len(syntax_errors) > 1 else '' + s = "s" if len(syntax_errors) > 1 else "" s_errors = "" for error in syntax_errors: s_errors += "{}\n".format(error[6:]) result = "Syntax error{}: \n{}".format(s, s_errors[:-1]) else: result = "Syntax valid" - return {'results': [{'types': mispattributes['output'], 'values': result}]} + return {"results": [{"types": mispattributes["output"], "values": result}]} def introspection(): @@ -52,5 +56,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/threatcrowd.py b/misp_modules/modules/expansion/threatcrowd.py index 4c7951611..d72fc3c32 100644 --- a/misp_modules/modules/expansion/threatcrowd.py +++ b/misp_modules/modules/expansion/threatcrowd.py @@ -1,25 +1,57 @@ import json -import requests import re -misperrors = {'error': 'Error'} -mispattributes = {'input': ["hostname", "domain", "ip-src", "ip-dst", "md5", "sha1", "sha256", "sha512", "whois-registrant-email"], - 'output': ["domain", "ip-src", "ip-dst", "text", "md5", "sha1", "sha256", "sha512", "hostname", "whois-registrant-email"] - } +import requests + +misperrors = {"error": "Error"} +mispattributes = { + "input": [ + "hostname", + "domain", + "ip-src", + "ip-dst", + "md5", + "sha1", + "sha256", + "sha512", + "whois-registrant-email", + ], + "output": [ + "domain", + "ip-src", + "ip-dst", + "text", + "md5", + "sha1", + "sha256", + "sha512", + "hostname", + "whois-registrant-email", + ], +} # possible module-types: 'expansion', 'hover' or both moduleinfo = { - 'version': '1', - 'author': 'chrisdoman', - 'description': 'Module to get information from ThreatCrowd.', - 'module-type': ['expansion'], - 'name': 'ThreatCrowd Lookup', - 'logo': 'threatcrowd.png', - 'requirements': [], - 'features': 'This module takes a MISP attribute as input and queries ThreatCrowd with it.\n\nThe result of this query is then parsed and some data is mapped into MISP attributes in order to enrich the input attribute.', - 'references': ['https://www.threatcrowd.org/'], - 'input': 'A MISP attribute included in the following list:\n- hostname\n- domain\n- ip-src\n- ip-dst\n- md5\n- sha1\n- sha256\n- sha512\n- whois-registrant-email', - 'output': 'MISP attributes mapped from the result of the query on ThreatCrowd, included in the following list:\n- domain\n- ip-src\n- ip-dst\n- text\n- md5\n- sha1\n- sha256\n- sha512\n- hostname\n- whois-registrant-email', + "version": "1", + "author": "chrisdoman", + "description": "Module to get information from ThreatCrowd.", + "module-type": ["expansion"], + "name": "ThreatCrowd Lookup", + "logo": "threatcrowd.png", + "requirements": [], + "features": ( + "This module takes a MISP attribute as input and queries ThreatCrowd with it.\n\nThe result of this query is" + " then parsed and some data is mapped into MISP attributes in order to enrich the input attribute." + ), + "references": ["https://www.threatcrowd.org/"], + "input": ( + "A MISP attribute included in the following list:\n- hostname\n- domain\n- ip-src\n- ip-dst\n- md5\n- sha1\n-" + " sha256\n- sha512\n- whois-registrant-email" + ), + "output": ( + "MISP attributes mapped from the result of the query on ThreatCrowd, included in the following list:\n-" + " domain\n- ip-src\n- ip-dst\n- text\n- md5\n- sha1\n- sha256\n- sha512\n- hostname\n- whois-registrant-email" + ), } moduleconfig = [] @@ -27,7 +59,7 @@ # Avoid adding windows update to enrichment etc. def isBlacklisted(value): - blacklist = ['8.8.8.8', '255.255.255.255', '192.168.56.', 'time.windows.com'] + blacklist = ["8.8.8.8", "255.255.255.255", "192.168.56.", "time.windows.com"] for b in blacklist: if value in b: @@ -68,18 +100,18 @@ def handler(q=False): r["results"] += getIP(q["ip-dst"]) if "domain" in q: r["results"] += getDomain(q["domain"]) - if 'hostname' in q: - r["results"] += getDomain(q['hostname']) - if 'md5' in q: - r["results"] += getHash(q['md5']) - if 'sha1' in q: - r["results"] += getHash(q['sha1']) - if 'sha256' in q: - r["results"] += getHash(q['sha256']) - if 'sha512' in q: - r["results"] += getHash(q['sha512']) - if 'whois-registrant-email' in q: - r["results"] += getEmail(q['whois-registrant-email']) + if "hostname" in q: + r["results"] += getDomain(q["hostname"]) + if "md5" in q: + r["results"] += getHash(q["md5"]) + if "sha1" in q: + r["results"] += getHash(q["sha1"]) + if "sha256" in q: + r["results"] += getHash(q["sha256"]) + if "sha512" in q: + r["results"] += getHash(q["sha512"]) + if "whois-registrant-email" in q: + r["results"] += getEmail(q["whois-registrant-email"]) uniq = [] for res in r["results"]: @@ -166,5 +198,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/threatfox.py b/misp_modules/modules/expansion/threatfox.py index ee82e4fd2..125ca9069 100644 --- a/misp_modules/modules/expansion/threatfox.py +++ b/misp_modules/modules/expansion/threatfox.py @@ -1,21 +1,34 @@ # -*- coding: utf-8 -*- -import requests import json -misperrors = {'error': 'Error'} -mispattributes = {'input': ['md5', 'sha1', 'sha256', 'domain', 'url', 'email-src', 'ip-dst|port', 'ip-src|port'], 'output': ['text']} +import requests + +misperrors = {"error": "Error"} +mispattributes = { + "input": [ + "md5", + "sha1", + "sha256", + "domain", + "url", + "email-src", + "ip-dst|port", + "ip-src|port", + ], + "output": ["text"], +} moduleinfo = { - 'version': '0.1', - 'author': 'Corsin Camichel', - 'description': 'Module to search for an IOC on ThreatFox by abuse.ch.', - 'module-type': ['hover', 'expansion'], - 'name': 'ThreadFox Lookup', - 'logo': '', - 'requirements': [], - 'features': '', - 'references': [], - 'input': '', - 'output': '', + "version": "0.1", + "author": "Corsin Camichel", + "description": "Module to search for an IOC on ThreatFox by abuse.ch.", + "module-type": ["hover", "expansion"], + "name": "ThreadFox Lookup", + "logo": "", + "requirements": [], + "features": "", + "references": [], + "input": "", + "output": "", } moduleconfig = [] @@ -47,21 +60,33 @@ def handler(q=False): request = json.loads(q) ret_val = "" - for input_type in mispattributes['input']: + for input_type in mispattributes["input"]: if input_type in request: to_query = request[input_type] break else: - misperrors['error'] = "Unsupported attributes type:" + misperrors["error"] = "Unsupported attributes type:" return misperrors data = {"query": "search_ioc", "search_term": f"{to_query}"} response = requests.post(API_URL, data=json.dumps(data)) if response.status_code == 200: result = json.loads(response.text) - if(result["query_status"] == "ok"): + if result["query_status"] == "ok": confidence_tag = confidence_level_to_tag(result["data"][0]["confidence_level"]) - ret_val = {'results': [{'types': mispattributes['output'], 'values': [result["data"][0]["threat_type_desc"]], 'tags': [result["data"][0]["malware"], result["data"][0]["malware_printable"], confidence_tag]}]} + ret_val = { + "results": [ + { + "types": mispattributes["output"], + "values": [result["data"][0]["threat_type_desc"]], + "tags": [ + result["data"][0]["malware"], + result["data"][0]["malware_printable"], + confidence_tag, + ], + } + ] + } return ret_val @@ -71,5 +96,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/threatminer.py b/misp_modules/modules/expansion/threatminer.py index 090f1fa1e..05fc53930 100755 --- a/misp_modules/modules/expansion/threatminer.py +++ b/misp_modules/modules/expansion/threatminer.py @@ -1,37 +1,78 @@ import json -import requests from collections import defaultdict -misperrors = {'error': 'Error'} -mispattributes = {'input': ['hostname', 'domain', 'ip-src', 'ip-dst', 'md5', 'sha1', 'sha256', 'sha512'], - 'output': ['domain', 'ip-src', 'ip-dst', 'text', 'md5', 'sha1', 'sha256', 'sha512', 'ssdeep', - 'authentihash', 'filename', 'whois-registrant-email', 'url', 'link'] - } +import requests + +misperrors = {"error": "Error"} +mispattributes = { + "input": [ + "hostname", + "domain", + "ip-src", + "ip-dst", + "md5", + "sha1", + "sha256", + "sha512", + ], + "output": [ + "domain", + "ip-src", + "ip-dst", + "text", + "md5", + "sha1", + "sha256", + "sha512", + "ssdeep", + "authentihash", + "filename", + "whois-registrant-email", + "url", + "link", + ], +} # possible module-types: 'expansion', 'hover' or both moduleinfo = { - 'version': '1', - 'author': 'KX499', - 'description': 'Module to get information from ThreatMiner.', - 'module-type': ['expansion'], - 'name': 'ThreatMiner Lookup', - 'logo': 'threatminer.png', - 'requirements': [], - 'features': 'This module takes a MISP attribute as input and queries ThreatMiner with it.\n\nThe result of this query is then parsed and some data is mapped into MISP attributes in order to enrich the input attribute.', - 'references': ['https://www.threatminer.org/'], - 'input': 'A MISP attribute included in the following list:\n- hostname\n- domain\n- ip-src\n- ip-dst\n- md5\n- sha1\n- sha256\n- sha512', - 'output': 'MISP attributes mapped from the result of the query on ThreatMiner, included in the following list:\n- domain\n- ip-src\n- ip-dst\n- text\n- md5\n- sha1\n- sha256\n- sha512\n- ssdeep\n- authentihash\n- filename\n- whois-registrant-email\n- url\n- link', + "version": "1", + "author": "KX499", + "description": "Module to get information from ThreatMiner.", + "module-type": ["expansion"], + "name": "ThreatMiner Lookup", + "logo": "threatminer.png", + "requirements": [], + "features": ( + "This module takes a MISP attribute as input and queries ThreatMiner with it.\n\nThe result of this query is" + " then parsed and some data is mapped into MISP attributes in order to enrich the input attribute." + ), + "references": ["https://www.threatminer.org/"], + "input": ( + "A MISP attribute included in the following list:\n- hostname\n- domain\n- ip-src\n- ip-dst\n- md5\n- sha1\n-" + " sha256\n- sha512" + ), + "output": ( + "MISP attributes mapped from the result of the query on ThreatMiner, included in the following list:\n-" + " domain\n- ip-src\n- ip-dst\n- text\n- md5\n- sha1\n- sha256\n- sha512\n- ssdeep\n- authentihash\n-" + " filename\n- whois-registrant-email\n- url\n- link" + ), } -class ThreatMiner(): +class ThreatMiner: def __init__(self): self.results = defaultdict(set) - self.comment = '{}: Threatminer - {}' - self.types_mapping = {'domain': '_get_domain', 'hostname': '_get_domain', - 'ip-dst': '_get_ip', 'ip-src': '_get_ip', - 'md5': '_get_hash', 'sha1': '_get_hash', - 'sha256': '_get_hash', 'sha512': '_get_hash'} + self.comment = "{}: Threatminer - {}" + self.types_mapping = { + "domain": "_get_domain", + "hostname": "_get_domain", + "ip-dst": "_get_ip", + "ip-src": "_get_ip", + "md5": "_get_hash", + "sha1": "_get_hash", + "sha256": "_get_hash", + "sha512": "_get_hash", + } @property def parsed_results(self): @@ -40,8 +81,13 @@ def parsed_results(self): if values: input_value, comment = key[:2] types = [k for k in key[2:]] - to_return.append({'types': types, 'values': list(values), - 'comment': self.comment.format(input_value, comment)}) + to_return.append( + { + "types": types, + "values": list(values), + "comment": self.comment.format(input_value, comment), + } + ) return to_return def parse_query(self, request): @@ -50,87 +96,118 @@ def parse_query(self, request): getattr(self, to_call)(request[input_type]) def _get_domain(self, q): - queries_mapping = {1: ('_add_whois', 'whois'), 2: ('_add_ip', 'pdns'), - 3: ('_add_uri', 'uri'), 4: ('_add_hash', 'samples'), - 5: ('_add_domain', 'subdomain'), 6: ('_add_link', 'report')} + queries_mapping = { + 1: ("_add_whois", "whois"), + 2: ("_add_ip", "pdns"), + 3: ("_add_uri", "uri"), + 4: ("_add_hash", "samples"), + 5: ("_add_domain", "subdomain"), + 6: ("_add_link", "report"), + } for flag, mapped in queries_mapping.items(): - req = requests.get('https://www.threatminer.org/domain.php', params={'q': q, 'api': 'True', 'rt': flag}) + req = requests.get( + "https://www.threatminer.org/domain.php", + params={"q": q, "api": "True", "rt": flag}, + ) if not req.status_code == 200: continue - results = req.json().get('results') + results = req.json().get("results") if not results: continue to_call, comment = mapped getattr(self, to_call)(results, q, comment) def _get_hash(self, q): - queries_mapping = {1: ('_add_filename', 'file'), 3: ('_add_network', 'network'), - 6: ('_add_text', 'detection'), 7: ('_add_hash', 'report')} + queries_mapping = { + 1: ("_add_filename", "file"), + 3: ("_add_network", "network"), + 6: ("_add_text", "detection"), + 7: ("_add_hash", "report"), + } for flag, mapped in queries_mapping.items(): - req = requests.get('https://www.threatminer.org/sample.php', params={'q': q, 'api': 'True', 'rt': flag}) + req = requests.get( + "https://www.threatminer.org/sample.php", + params={"q": q, "api": "True", "rt": flag}, + ) if not req.status_code == 200: continue - results = req.json().get('results') + results = req.json().get("results") if not results: continue to_call, comment = mapped getattr(self, to_call)(results, q, comment) def _get_ip(self, q): - queries_mapping = {1: ('_add_whois', 'whois'), 2: ('_add_ip', 'pdns'), - 3: ('_add_uri', 'uri'), 4: ('_add_hash', 'samples'), - 5: ('_add_x509', 'ssl'), 6: ('_add_link', 'report')} + queries_mapping = { + 1: ("_add_whois", "whois"), + 2: ("_add_ip", "pdns"), + 3: ("_add_uri", "uri"), + 4: ("_add_hash", "samples"), + 5: ("_add_x509", "ssl"), + 6: ("_add_link", "report"), + } for flag, mapped in queries_mapping.items(): - req = requests.get('https://www.threatminer.org/host.php', params={'q': q, 'api': 'True', 'rt': flag}) + req = requests.get( + "https://www.threatminer.org/host.php", + params={"q": q, "api": "True", "rt": flag}, + ) if not req.status_code == 200: continue - results = req.json().get('results') + results = req.json().get("results") if not results: continue to_call, comment = mapped getattr(self, to_call)(results, q, comment) def _add_domain(self, results, q, comment): - self.results[(q, comment, 'domain')].update({result for result in results if isinstance(result, str)}) + self.results[(q, comment, "domain")].update({result for result in results if isinstance(result, str)}) def _add_filename(self, results, q, comment): - self.results[(q, comment, 'filename')].update({result['file_name'] for result in results if result.get('file_name')}) + self.results[(q, comment, "filename")].update( + {result["file_name"] for result in results if result.get("file_name")} + ) def _add_hash(self, results, q, comment): - self.results[(q, comment, 'sha256')].update({result for result in results if isinstance(result, str)}) + self.results[(q, comment, "sha256")].update({result for result in results if isinstance(result, str)}) def _add_ip(self, results, q, comment): - self.results[(q, comment, 'ip-src', 'ip-dst')].update({result['ip'] for result in results if result.get('ip')}) + self.results[(q, comment, "ip-src", "ip-dst")].update({result["ip"] for result in results if result.get("ip")}) def _add_link(self, results, q, comment): - self.results[(q, comment, 'link')].update({result['URL'] for result in results if result.get('URL')}) + self.results[(q, comment, "link")].update({result["URL"] for result in results if result.get("URL")}) def _add_network(self, results, q, comment): for result in results: - domains = result.get('domains') + domains = result.get("domains") if domains: - self.results[(q, comment, 'domain')].update({domain['domain'] for domain in domains if domain.get('domain')}) - hosts = result.get('hosts') + self.results[(q, comment, "domain")].update( + {domain["domain"] for domain in domains if domain.get("domain")} + ) + hosts = result.get("hosts") if hosts: - self.results[(q, comment, 'ip-src', 'ip-dst')].update({host for host in hosts if isinstance(host, str)}) + self.results[(q, comment, "ip-src", "ip-dst")].update({host for host in hosts if isinstance(host, str)}) def _add_text(self, results, q, comment): for result in results: - detections = result.get('av_detections') + detections = result.get("av_detections") if detections: - self.results[(q, comment, 'text')].update({d['detection'] for d in detections if d.get('detection')}) + self.results[(q, comment, "text")].update({d["detection"] for d in detections if d.get("detection")}) def _add_uri(self, results, q, comment): - self.results[(q, comment, 'url')].update({result['uri'] for result in results if result.get('uri')}) + self.results[(q, comment, "url")].update({result["uri"] for result in results if result.get("uri")}) def _add_whois(self, results, q, comment): for result in results: - emails = result.get('whois', {}).get('emails') + emails = result.get("whois", {}).get("emails") if emails: - self.results[(q, comment, 'whois-registrant-email')].update({email for em_type, email in emails.items() if em_type == 'registrant' and email}) + self.results[(q, comment, "whois-registrant-email")].update( + {email for em_type, email in emails.items() if em_type == "registrant" and email} + ) def _add_x509(self, results, q, comment): - self.results[(q, comment, 'x509-fingerprint-sha1')].update({result for result in results if isinstance(result, str)}) + self.results[(q, comment, "x509-fingerprint-sha1")].update( + {result for result in results if isinstance(result, str)} + ) def handler(q=False): @@ -141,7 +218,7 @@ def handler(q=False): parser = ThreatMiner() parser.parse_query(q) - return {'results': parser.parsed_results} + return {"results": parser.parsed_results} def introspection(): diff --git a/misp_modules/modules/expansion/triage_submit.py b/misp_modules/modules/expansion/triage_submit.py index 97db16c92..c19414631 100644 --- a/misp_modules/modules/expansion/triage_submit.py +++ b/misp_modules/modules/expansion/triage_submit.py @@ -1,21 +1,22 @@ -import json -import requests import base64 import io +import json import zipfile -misperrors = {'error': 'Error'} -mispattributes = {'input': ['attachment', 'malware-sample', 'url'], 'output': ['link']} +import requests + +misperrors = {"error": "Error"} +mispattributes = {"input": ["attachment", "malware-sample", "url"], "output": ["link"]} moduleinfo = { - 'version': '1', - 'author': 'Karen Yousefi', - 'description': 'Module to submit samples to tria.ge', - 'module-type': ['expansion', 'hover'], - 'name': 'Triage Submit', - 'logo': '', + "version": "1", + "author": "Karen Yousefi", + "description": "Module to submit samples to tria.ge", + "module-type": ["expansion", "hover"], + "name": "Triage Submit", + "logo": "", } -moduleconfig = ['apikey', 'url_mode'] +moduleconfig = ["apikey", "url_mode"] def handler(q=False): @@ -24,28 +25,28 @@ def handler(q=False): request = json.loads(q) - if request.get('config', {}).get('apikey') is None: - misperrors['error'] = 'tria.ge API key is missing' + if request.get("config", {}).get("apikey") is None: + misperrors["error"] = "tria.ge API key is missing" return misperrors - api_key = request['config']['apikey'] - url_mode = request['config'].get('url_mode', 'submit') # 'submit' or 'fetch' - base_url = 'https://tria.ge/api/v0/samples' - headers = {'Authorization': f'Bearer {api_key}'} + api_key = request["config"]["apikey"] + url_mode = request["config"].get("url_mode", "submit") # 'submit' or 'fetch' + base_url = "https://tria.ge/api/v0/samples" + headers = {"Authorization": f"Bearer {api_key}"} - if 'attachment' in request: - data = request['data'] - filename = request['attachment'] + if "attachment" in request: + data = request["data"] + filename = request["attachment"] return submit_file(headers, base_url, data, filename) - elif 'malware-sample' in request: - data = request['data'] - filename = request['malware-sample'].split('|')[0] + elif "malware-sample" in request: + data = request["data"] + filename = request["malware-sample"].split("|")[0] return submit_file(headers, base_url, data, filename, is_malware_sample=True) - elif 'url' in request: - url = request['url'] + elif "url" in request: + url = request["url"] return submit_url(headers, base_url, url, url_mode) else: - misperrors['error'] = 'Unsupported input type' + misperrors["error"] = "Unsupported input type" return misperrors @@ -54,59 +55,59 @@ def submit_file(headers, base_url, data, filename, is_malware_sample=False): if is_malware_sample: file_data = base64.b64decode(data) zip_file = zipfile.ZipFile(io.BytesIO(file_data)) - file_data = zip_file.read(zip_file.namelist()[0], pwd=b'infected') + file_data = zip_file.read(zip_file.namelist()[0], pwd=b"infected") else: file_data = base64.b64decode(data) - files = {'file': (filename, file_data)} + files = {"file": (filename, file_data)} response = requests.post(base_url, headers=headers, files=files) response.raise_for_status() result = response.json() - sample_id = result['id'] - sample_url = f'https://tria.ge/{sample_id}' + sample_id = result["id"] + sample_url = f"https://tria.ge/{sample_id}" return { - 'results': [ + "results": [ { - 'types': 'link', - 'values': sample_url, - 'comment': 'Link to tria.ge analysis', + "types": "link", + "values": sample_url, + "comment": "Link to tria.ge analysis", } ] } except Exception as e: - misperrors['error'] = f'Error submitting to tria.ge: {str(e)}' + misperrors["error"] = f"Error submitting to tria.ge: {str(e)}" return misperrors def submit_url(headers, base_url, url, mode): try: - if mode == 'fetch': - data = {'kind': 'fetch', 'url': url} + if mode == "fetch": + data = {"kind": "fetch", "url": url} else: # submit - data = {'kind': 'url', 'url': url} + data = {"kind": "url", "url": url} response = requests.post(base_url, headers=headers, json=data) response.raise_for_status() result = response.json() - sample_id = result['id'] - sample_url = f'https://tria.ge/{sample_id}' + sample_id = result["id"] + sample_url = f"https://tria.ge/{sample_id}" return { - 'results': [ + "results": [ { - 'types': 'link', - 'values': sample_url, - 'comment': f'Link to tria.ge analysis ({mode} mode)', + "types": "link", + "values": sample_url, + "comment": f"Link to tria.ge analysis ({mode} mode)", } ] } except Exception as e: - misperrors['error'] = f'Error submitting to tria.ge: {str(e)}' + misperrors["error"] = f"Error submitting to tria.ge: {str(e)}" return misperrors @@ -115,5 +116,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/trustar_enrich.py b/misp_modules/modules/expansion/trustar_enrich.py index 6cbdcb3ef..9e6a8790f 100644 --- a/misp_modules/modules/expansion/trustar_enrich.py +++ b/misp_modules/modules/expansion/trustar_enrich.py @@ -1,29 +1,54 @@ import json -import pymisp from base64 import b64encode from collections import OrderedDict -from . import check_input_attribute, checking_error, standard_error_message -from pymisp import MISPAttribute, MISPEvent, MISPObject -from trustar import TruStar, Indicator from urllib.parse import quote -misperrors = {'error': "Error"} +import pymisp +from pymisp import MISPAttribute, MISPEvent, MISPObject +from trustar import Indicator, TruStar + +from . import check_input_attribute, checking_error, standard_error_message + +misperrors = {"error": "Error"} mispattributes = { - 'input': ["btc", "domain", "email-src", "filename", "hostname", "ip-src", "ip-dst", "malware-type", "md5", "sha1", - "sha256", "url"], 'format': 'misp_standard'} + "input": [ + "btc", + "domain", + "email-src", + "filename", + "hostname", + "ip-src", + "ip-dst", + "malware-type", + "md5", + "sha1", + "sha256", + "url", + ], + "format": "misp_standard", +} moduleinfo = { - 'version': '0.1', - 'author': 'Jesse Hedden', - 'description': 'Module to get enrich indicators with TruSTAR.', - 'module-type': ['hover', 'expansion'], - 'name': 'TruSTAR Enrich', - 'logo': 'trustar.png', - 'requirements': [], - 'features': 'This module enriches MISP attributes with scoring and metadata from TruSTAR.\n\nThe TruSTAR indicator summary is appended to the attributes along with links to any associated reports.', - 'references': ['https://docs.trustar.co/api/v13/indicators/get_indicator_summaries.html'], - 'input': 'Any of the following MISP attributes:\n- btc\n- domain\n- email-src\n- filename\n- hostname\n- ip-src\n- ip-dst\n- md5\n- sha1\n- sha256\n- url', - 'output': 'MISP attributes enriched with indicator summary data from the TruSTAR API. Data includes a severity level score and additional source and scoring info.', + "version": "0.1", + "author": "Jesse Hedden", + "description": "Module to get enrich indicators with TruSTAR.", + "module-type": ["hover", "expansion"], + "name": "TruSTAR Enrich", + "logo": "trustar.png", + "requirements": [], + "features": ( + "This module enriches MISP attributes with scoring and metadata from TruSTAR.\n\nThe TruSTAR indicator summary" + " is appended to the attributes along with links to any associated reports." + ), + "references": ["https://docs.trustar.co/api/v13/indicators/get_indicator_summaries.html"], + "input": ( + "Any of the following MISP attributes:\n- btc\n- domain\n- email-src\n- filename\n- hostname\n- ip-src\n-" + " ip-dst\n- md5\n- sha1\n- sha256\n- url" + ), + "output": ( + "MISP attributes enriched with indicator summary data from the TruSTAR API. Data includes a severity level" + " score and additional source and scoring info." + ), } moduleconfig = ["user_api_key", "user_api_secret", "enclave_ids"] @@ -33,18 +58,18 @@ class TruSTARParser: ENTITY_TYPE_MAPPINGS = { - 'BITCOIN_ADDRESS': "btc", - 'CIDR_BLOCK': "ip-src", - 'CVE': "vulnerability", - 'URL': "url", - 'EMAIL_ADDRESS': "email-src", - 'SOFTWARE': "filename", - 'IP': "ip-src", - 'MALWARE': "malware-type", - 'MD5': "md5", - 'REGISTRY_KEY': "regkey", - 'SHA1': "sha1", - 'SHA256': "sha256" + "BITCOIN_ADDRESS": "btc", + "CIDR_BLOCK": "ip-src", + "CVE": "vulnerability", + "URL": "url", + "EMAIL_ADDRESS": "email-src", + "SOFTWARE": "filename", + "IP": "ip-src", + "MALWARE": "malware-type", + "MD5": "md5", + "REGISTRY_KEY": "regkey", + "SHA1": "sha1", + "SHA256": "sha256", } # Relevant fields from each TruSTAR endpoint @@ -56,8 +81,8 @@ class TruSTARParser: CLIENT_METATAG = f"MISP-{pymisp.__version__}" def __init__(self, attribute, config): - config['enclave_ids'] = config.get('enclave_ids', "").strip().split(',') - config['client_metatag'] = self.CLIENT_METATAG + config["enclave_ids"] = config.get("enclave_ids", "").strip().split(",") + config["client_metatag"] = self.CLIENT_METATAG self.ts_client = TruStar(config=config) self.misp_event = MISPEvent() @@ -71,10 +96,10 @@ def get_results(self): """ try: event = json.loads(self.misp_event.to_json()) - results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} - return {'results': results} + results = {key: event[key] for key in ("Attribute", "Object") if (key in event and event[key])} + return {"results": results} except Exception as e: - misperrors['error'] += f" -- Encountered issue serializing enrichment data -- {e}" + misperrors["error"] += f" -- Encountered issue serializing enrichment data -- {e}" return misperrors def generate_trustar_link(self, entity_type, entity_value): @@ -99,8 +124,8 @@ def extract_tags(enrichment_report): :param: Enrichment data. :return: List of tags. """ - if enrichment_report and enrichment_report.get('tags'): - return [tag.get('name') for tag in enrichment_report.pop('tags')] + if enrichment_report and enrichment_report.get("tags"): + return [tag.get("name") for tag in enrichment_report.pop("tags")] return None def generate_enrichment_report(self, summary, metadata): @@ -118,12 +143,14 @@ def generate_enrichment_report(self, summary, metadata): if summary: summary_dict = summary.to_dict() enrichment_report.update( - {field: summary_dict[field] for field in self.SUMMARY_FIELDS if summary_dict.get(field)}) + {field: summary_dict[field] for field in self.SUMMARY_FIELDS if summary_dict.get(field)} + ) if metadata: metadata_dict = metadata.to_dict() enrichment_report.update( - {field: metadata_dict[field] for field in self.METADATA_FIELDS if metadata_dict.get(field)}) + {field: metadata_dict[field] for field in self.METADATA_FIELDS if metadata_dict.get(field)} + ) return enrichment_report @@ -143,7 +170,7 @@ def parse_indicator_summary(self, indicator, summary, metadata): elif metadata and metadata.type in self.ENTITY_TYPE_MAPPINGS: indicator_type = metadata.type else: - misperrors['error'] += " -- Attribute not found or not supported" + misperrors["error"] += " -- Attribute not found or not supported" raise Exception try: @@ -153,11 +180,17 @@ def parse_indicator_summary(self, indicator, summary, metadata): if enrichment_report: # Create MISP trustar_report object and populate it with enrichment data - trustar_obj = MISPObject('trustar_report') - trustar_obj.add_attribute(indicator_type, attribute_type=self.ENTITY_TYPE_MAPPINGS[indicator_type], - value=indicator) - trustar_obj.add_attribute("INDICATOR_SUMMARY", attribute_type="text", - value=json.dumps(enrichment_report, indent=4)) + trustar_obj = MISPObject("trustar_report") + trustar_obj.add_attribute( + indicator_type, + attribute_type=self.ENTITY_TYPE_MAPPINGS[indicator_type], + value=indicator, + ) + trustar_obj.add_attribute( + "INDICATOR_SUMMARY", + attribute_type="text", + value=json.dumps(enrichment_report, indent=4), + ) report_link = self.generate_trustar_link(indicator_type, indicator) trustar_obj.add_attribute("REPORT_LINK", attribute_type="link", value=report_link) @@ -172,7 +205,7 @@ def parse_indicator_summary(self, indicator, summary, metadata): self.misp_event.add_attribute_tag(tag, indicator) except Exception as e: - misperrors['error'] += f" -- Error enriching attribute {indicator} -- {e}" + misperrors["error"] += f" -- Error enriching attribute {indicator} -- {e}" raise e @@ -189,37 +222,38 @@ def handler(q=False): request = json.loads(q) - config = request.get('config', {}) - if not config.get('user_api_key') or not config.get('user_api_secret'): - misperrors['error'] = "Your TruSTAR API key and secret are required for indicator enrichment." + config = request.get("config", {}) + if not config.get("user_api_key") or not config.get("user_api_secret"): + misperrors["error"] = "Your TruSTAR API key and secret are required for indicator enrichment." return misperrors - if not request.get('attribute') or not check_input_attribute(request['attribute'], requirements=('type', 'value')): - return {'error': f'{standard_error_message}, {checking_error}.'} - attribute = request['attribute'] - if attribute['type'] not in mispattributes['input']: - return {'error': 'Unsupported attribute type.'} + if not request.get("attribute") or not check_input_attribute(request["attribute"], requirements=("type", "value")): + return {"error": f"{standard_error_message}, {checking_error}."} + attribute = request["attribute"] + if attribute["type"] not in mispattributes["input"]: + return {"error": "Unsupported attribute type."} trustar_parser = TruSTARParser(attribute, config) metadata = None summary = None try: - metadata = trustar_parser.ts_client.get_indicators_metadata([Indicator(value=attribute['value'])])[0] + metadata = trustar_parser.ts_client.get_indicators_metadata([Indicator(value=attribute["value"])])[0] except IndexError: - misperrors['error'] += f" -- No metadata found for indicator {attribute['value']}" + misperrors["error"] += f" -- No metadata found for indicator {attribute['value']}" except Exception as e: - misperrors['error'] += f" -- Could not retrieve indicator metadata from TruSTAR {e}" + misperrors["error"] += f" -- Could not retrieve indicator metadata from TruSTAR {e}" try: - summary = list( - trustar_parser.ts_client.get_indicator_summaries([attribute['value']], page_size=MAX_PAGE_SIZE))[0] + summary = list(trustar_parser.ts_client.get_indicator_summaries([attribute["value"]], page_size=MAX_PAGE_SIZE))[ + 0 + ] except IndexError: - misperrors['error'] += f" -- No summary data found for indicator {attribute['value']}" + misperrors["error"] += f" -- No summary data found for indicator {attribute['value']}" except Exception as e: - misperrors['error'] += f" -- Unable to retrieve TruSTAR summary data: {e}" + misperrors["error"] += f" -- Unable to retrieve TruSTAR summary data: {e}" try: - trustar_parser.parse_indicator_summary(attribute['value'], summary, metadata) + trustar_parser.parse_indicator_summary(attribute["value"], summary, metadata) except Exception: return misperrors @@ -231,5 +265,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/urlhaus.py b/misp_modules/modules/expansion/urlhaus.py index 8c7efa2ec..f04a7e1f6 100644 --- a/misp_modules/modules/expansion/urlhaus.py +++ b/misp_modules/modules/expansion/urlhaus.py @@ -1,56 +1,64 @@ # -*- coding: utf-8 -*- import json + import requests -from . import check_input_attribute, standard_error_message from pymisp import MISPAttribute, MISPEvent, MISPObject -misperrors = {'error': 'Error'} -mispattributes = {'input': ['domain', 'hostname', 'ip-src', 'ip-dst', 'md5', 'sha256', 'url'], - 'output': ['url', 'filename', 'md5', 'sha256'], - 'format': 'misp_standard'} +from . import check_input_attribute, standard_error_message + +misperrors = {"error": "Error"} +mispattributes = { + "input": ["domain", "hostname", "ip-src", "ip-dst", "md5", "sha256", "url"], + "output": ["url", "filename", "md5", "sha256"], + "format": "misp_standard", +} moduleinfo = { - 'version': '0.1', - 'author': 'Christian Studer', - 'description': 'Query of the URLhaus API to get additional information about the input attribute.', - 'module-type': ['expansion', 'hover'], - 'name': 'URLhaus Lookup', - 'logo': 'urlhaus.png', - 'requirements': [], - 'features': 'Module using the new format of modules able to return attributes and objects.\n\nThe module takes one of the attribute type specified as input, and query the URLhaus API with it. If any result is returned by the API, attributes and objects are created accordingly.', - 'references': ['https://urlhaus.abuse.ch/'], - 'input': 'A domain, hostname, url, ip, md5 or sha256 attribute.', - 'output': 'MISP attributes & objects fetched from the result of the URLhaus API query.', + "version": "0.1", + "author": "Christian Studer", + "description": "Query of the URLhaus API to get additional information about the input attribute.", + "module-type": ["expansion", "hover"], + "name": "URLhaus Lookup", + "logo": "urlhaus.png", + "requirements": [], + "features": ( + "Module using the new format of modules able to return attributes and objects.\n\nThe module takes one of the" + " attribute type specified as input, and query the URLhaus API with it. If any result is returned by the API," + " attributes and objects are created accordingly." + ), + "references": ["https://urlhaus.abuse.ch/"], + "input": "A domain, hostname, url, ip, md5 or sha256 attribute.", + "output": "MISP attributes & objects fetched from the result of the URLhaus API query.", } moduleconfig = [] -file_keys = ('filename', 'response_size', 'response_md5', 'response_sha256') -file_relations = ('filename', 'size-in-bytes', 'md5', 'sha256') -vt_keys = ('result', 'link') -vt_types = ('text', 'link') -vt_relations = ('detection-ratio', 'permalink') +file_keys = ("filename", "response_size", "response_md5", "response_sha256") +file_relations = ("filename", "size-in-bytes", "md5", "sha256") +vt_keys = ("result", "link") +vt_types = ("text", "link") +vt_relations = ("detection-ratio", "permalink") -class URLhaus(): +class URLhaus: def __init__(self): super(URLhaus, self).__init__() self.misp_event = MISPEvent() @staticmethod def _create_vt_object(virustotal): - vt_object = MISPObject('virustotal-report') + vt_object = MISPObject("virustotal-report") for key, vt_type, relation in zip(vt_keys, vt_types, vt_relations): - vt_object.add_attribute(relation, **{'type': vt_type, 'value': virustotal[key]}) + vt_object.add_attribute(relation, **{"type": vt_type, "value": virustotal[key]}) return vt_object def get_result(self): event = json.loads(self.misp_event.to_json()) - results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} - return {'results': results} + results = {key: event[key] for key in ("Attribute", "Object") if (key in event and event[key])} + return {"results": results} def parse_error(self, query_status): - if query_status == 'no_results': - return {'error': f'No results found on URLhaus for this {self.attribute.type} attribute'} - return {'error': f'Error encountered during the query of URLhaus: {query_status}'} + if query_status == "no_results": + return {"error": f"No results found on URLhaus for this {self.attribute.type} attribute"} + return {"error": f"Error encountered during the query of URLhaus: {query_status}"} class HostQuery(URLhaus): @@ -58,15 +66,15 @@ def __init__(self, attribute): super(HostQuery, self).__init__() self.attribute = MISPAttribute() self.attribute.from_dict(**attribute) - self.url = 'https://urlhaus-api.abuse.ch/v1/host/' + self.url = "https://urlhaus-api.abuse.ch/v1/host/" def query_api(self): - response = requests.post(self.url, data={'host': self.attribute.value}).json() - if response['query_status'] != 'ok': - return self.parse_error(response['query_status']) - if 'urls' in response and response['urls']: - for url in response['urls']: - self.misp_event.add_attribute(type='url', value=url['url']) + response = requests.post(self.url, data={"host": self.attribute.value}).json() + if response["query_status"] != "ok": + return self.parse_error(response["query_status"]) + if "urls" in response and response["urls"]: + for url in response["urls"]: + self.misp_event.add_attribute(type="url", value=url["url"]) return self.get_result() @@ -75,32 +83,39 @@ def __init__(self, attribute): super(PayloadQuery, self).__init__() self.attribute = MISPAttribute() self.attribute.from_dict(**attribute) - self.url = 'https://urlhaus-api.abuse.ch/v1/payload/' + self.url = "https://urlhaus-api.abuse.ch/v1/payload/" def query_api(self): hash_type = self.attribute.type - file_object = MISPObject('file') - if hasattr(self.attribute, 'object_id') and hasattr(self.attribute, 'event_id') and self.attribute.event_id != '0': + file_object = MISPObject("file") + if ( + hasattr(self.attribute, "object_id") + and hasattr(self.attribute, "event_id") + and self.attribute.event_id != "0" + ): file_object.id = self.attribute.object_id - response = requests.post(self.url, data={'{}_hash'.format(hash_type): self.attribute.value}).json() - if response['query_status'] != 'ok': - return self.parse_error(response['query_status']) - other_hash_type = 'md5' if hash_type == 'sha256' else 'sha256' - for key, relation in zip(('{}_hash'.format(other_hash_type), 'file_size'), (other_hash_type, 'size-in-bytes')): + response = requests.post(self.url, data={"{}_hash".format(hash_type): self.attribute.value}).json() + if response["query_status"] != "ok": + return self.parse_error(response["query_status"]) + other_hash_type = "md5" if hash_type == "sha256" else "sha256" + for key, relation in zip( + ("{}_hash".format(other_hash_type), "file_size"), + (other_hash_type, "size-in-bytes"), + ): if response[key]: - file_object.add_attribute(relation, **{'type': relation, 'value': response[key]}) - if response['virustotal']: - vt_object = self._create_vt_object(response['virustotal']) - file_object.add_reference(vt_object.uuid, 'analyzed-with') + file_object.add_attribute(relation, **{"type": relation, "value": response[key]}) + if response["virustotal"]: + vt_object = self._create_vt_object(response["virustotal"]) + file_object.add_reference(vt_object.uuid, "analyzed-with") self.misp_event.add_object(**vt_object) - _filename_ = 'filename' - for url in response['urls']: + _filename_ = "filename" + for url in response["urls"]: attribute = MISPAttribute() - attribute.from_dict(type='url', value=url['url']) + attribute.from_dict(type="url", value=url["url"]) self.misp_event.add_attribute(**attribute) - file_object.add_reference(attribute.uuid, 'retrieved-from') + file_object.add_reference(attribute.uuid, "retrieved-from") if url[_filename_]: - file_object.add_attribute(_filename_, **{'type': _filename_, 'value': url[_filename_]}) + file_object.add_attribute(_filename_, **{"type": _filename_, "value": url[_filename_]}) if any((file_object.attributes, file_object.references)): self.misp_event.add_object(**file_object) return self.get_result() @@ -111,47 +126,53 @@ def __init__(self, attribute): super(UrlQuery, self).__init__() self.attribute = MISPAttribute() self.attribute.from_dict(**attribute) - self.url = 'https://urlhaus-api.abuse.ch/v1/url/' + self.url = "https://urlhaus-api.abuse.ch/v1/url/" @staticmethod def _create_file_object(payload): - file_object = MISPObject('file') + file_object = MISPObject("file") for key, relation in zip(file_keys, file_relations): if payload[key]: - file_object.add_attribute(relation, **{'type': relation, 'value': payload[key]}) + file_object.add_attribute(relation, **{"type": relation, "value": payload[key]}) return file_object def query_api(self): - response = requests.post(self.url, data={'url': self.attribute.value}).json() - if response['query_status'] != 'ok': - return self.parse_error(response['query_status']) - if 'payloads' in response and response['payloads']: - for payload in response['payloads']: + response = requests.post(self.url, data={"url": self.attribute.value}).json() + if response["query_status"] != "ok": + return self.parse_error(response["query_status"]) + if "payloads" in response and response["payloads"]: + for payload in response["payloads"]: file_object = self._create_file_object(payload) - if payload['virustotal']: - vt_object = self._create_vt_object(payload['virustotal']) - file_object.add_reference(vt_object.uuid, 'analyzed-with') + if payload["virustotal"]: + vt_object = self._create_vt_object(payload["virustotal"]) + file_object.add_reference(vt_object.uuid, "analyzed-with") self.misp_event.add_object(**vt_object) if any((file_object.attributes, file_object.references)): self.misp_event.add_object(**file_object) return self.get_result() -_misp_type_mapping = {'url': UrlQuery, 'md5': PayloadQuery, 'sha256': PayloadQuery, - 'domain': HostQuery, 'hostname': HostQuery, - 'ip-src': HostQuery, 'ip-dst': HostQuery} +_misp_type_mapping = { + "url": UrlQuery, + "md5": PayloadQuery, + "sha256": PayloadQuery, + "domain": HostQuery, + "hostname": HostQuery, + "ip-src": HostQuery, + "ip-dst": HostQuery, +} def handler(q=False): if q is False: return False request = json.loads(q) - if not request.get('attribute') or not check_input_attribute(request['attribute']): - return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} - attribute = request['attribute'] - if attribute['type'] not in mispattributes['input']: - return {'error': 'Unsupported attribute type.'} - urlhaus_parser = _misp_type_mapping[attribute['type']](attribute) + if not request.get("attribute") or not check_input_attribute(request["attribute"]): + return {"error": f"{standard_error_message}, which should contain at least a type, a value and an uuid."} + attribute = request["attribute"] + if attribute["type"] not in mispattributes["input"]: + return {"error": "Unsupported attribute type."} + urlhaus_parser = _misp_type_mapping[attribute["type"]](attribute) return urlhaus_parser.query_api() @@ -160,5 +181,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/urlscan.py b/misp_modules/modules/expansion/urlscan.py index a523e559c..0526f89d1 100644 --- a/misp_modules/modules/expansion/urlscan.py +++ b/misp_modules/modules/expansion/urlscan.py @@ -1,36 +1,40 @@ import json -import requests import logging import sys import time -log = logging.getLogger('urlscan') +import requests + +log = logging.getLogger("urlscan") log.setLevel(logging.DEBUG) ch = logging.StreamHandler(sys.stdout) ch.setLevel(logging.DEBUG) -formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") ch.setFormatter(formatter) log.addHandler(ch) moduleinfo = { - 'version': '0.1', - 'author': 'Dave Johnson', - 'description': 'An expansion module to query urlscan.io.', - 'module-type': ['expansion'], - 'name': 'URLScan Lookup', - 'logo': 'urlscan.jpg', - 'requirements': ['An access to the urlscan.io API'], - 'features': 'This module takes a MISP attribute as input and queries urlscan.io with it.\n\nThe result of this query is then parsed and some data is mapped into MISP attributes in order to enrich the input attribute.', - 'references': ['https://urlscan.io/'], - 'input': 'A domain, hostname or url attribute.', - 'output': 'MISP attributes mapped from the result of the query on urlscan.io.', + "version": "0.1", + "author": "Dave Johnson", + "description": "An expansion module to query urlscan.io.", + "module-type": ["expansion"], + "name": "URLScan Lookup", + "logo": "urlscan.jpg", + "requirements": ["An access to the urlscan.io API"], + "features": ( + "This module takes a MISP attribute as input and queries urlscan.io with it.\n\nThe result of this query is" + " then parsed and some data is mapped into MISP attributes in order to enrich the input attribute." + ), + "references": ["https://urlscan.io/"], + "input": "A domain, hostname or url attribute.", + "output": "MISP attributes mapped from the result of the query on urlscan.io.", } -moduleconfig = ['apikey'] -misperrors = {'error': 'Error'} +moduleconfig = ["apikey"] +misperrors = {"error": "Error"} mispattributes = { - 'input': ['hostname', 'domain', 'ip-src', 'ip-dst', 'url'], - 'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'url', 'text', 'link', 'hash'] + "input": ["hostname", "domain", "ip-src", "ip-dst", "url"], + "output": ["hostname", "domain", "ip-src", "ip-dst", "url", "text", "link", "hash"], } @@ -38,174 +42,233 @@ def handler(q=False): if q is False: return False request = json.loads(q) - if not request.get('config') or not request['config'].get('apikey'): - misperrors['error'] = 'Urlscan apikey is missing' + if not request.get("config") or not request["config"].get("apikey"): + misperrors["error"] = "Urlscan apikey is missing" return misperrors - client = urlscanAPI(request['config']['apikey']) + client = urlscanAPI(request["config"]["apikey"]) - r = {'results': []} + r = {"results": []} - if 'ip-src' in request: - r['results'] += lookup_indicator(client, request['ip-src']) - if 'ip-dst' in request: - r['results'] += lookup_indicator(client, request['ip-dst']) - if 'domain' in request: - r['results'] += lookup_indicator(client, request['domain']) - if 'hostname' in request: - r['results'] += lookup_indicator(client, request['hostname']) - if 'url' in request: - r['results'] += lookup_indicator(client, request['url']) + if "ip-src" in request: + r["results"] += lookup_indicator(client, request["ip-src"]) + if "ip-dst" in request: + r["results"] += lookup_indicator(client, request["ip-dst"]) + if "domain" in request: + r["results"] += lookup_indicator(client, request["domain"]) + if "hostname" in request: + r["results"] += lookup_indicator(client, request["hostname"]) + if "url" in request: + r["results"] += lookup_indicator(client, request["url"]) # Return any errors generated from lookup to the UI and remove duplicates uniq = [] - log.debug(r['results']) - for item in r['results']: + log.debug(r["results"]) + for item in r["results"]: log.debug(item) - if 'error' in item: - misperrors['error'] = item['error'] + if "error" in item: + misperrors["error"] = item["error"] return misperrors if item not in uniq: uniq.append(item) - r['results'] = uniq + r["results"] = uniq return r def lookup_indicator(client, query): result = client.search_url(query) - log.debug('RESULTS: ' + json.dumps(result)) + log.debug("RESULTS: " + json.dumps(result)) r = [] misp_comment = "{}: Enriched via the urlscan module".format(query) # Determine if the page is reachable - for request in result['data']['requests']: - if request['response'].get('failed'): - if request['response']['failed']['errorText']: - if request['response']['failed']['errorText'] in ["net::ERR_ABORTED", "net::ERR_FAILED", "net::ERR_QUIC_PROTOCOL_ERROR"]: + for request in result["data"]["requests"]: + if request["response"].get("failed"): + if request["response"]["failed"]["errorText"]: + if request["response"]["failed"]["errorText"] in [ + "net::ERR_ABORTED", + "net::ERR_FAILED", + "net::ERR_QUIC_PROTOCOL_ERROR", + ]: continue - log.debug('The page could not load') + log.debug("The page could not load") r.append( - {'error': 'Domain could not be resolved: {}'.format(request['response']['failed']['errorText'])}) - - if result.get('page'): - if result['page'].get('domain'): - misp_val = result['page']['domain'] - r.append({'types': 'domain', - 'categories': ['Network activity'], - 'values': misp_val, - 'comment': f"{misp_comment} - Domain"}) - - if result['page'].get('ip'): - misp_val = result['page']['ip'] - r.append({'types': 'ip-dst', - 'categories': ['Network activity'], - 'values': misp_val, - 'comment': f"{misp_comment} - IP"}) - - if result['page'].get('ptr'): - misp_val = result['page']['ptr'] - r.append({'types': 'hostname', - 'categories': ['Network activity'], - 'values': misp_val, - 'comment': f"{misp_comment} - PTR"}) - - if result['page'].get('country'): - misp_val = 'country: ' + result['page']['country'] - if result['page'].get('city'): - misp_val += ', city: ' + result['page']['city'] - r.append({'types': 'text', - 'categories': ['External analysis'], - 'values': misp_val, - 'comment': f"{misp_comment} - Country/City"}) - - if result['page'].get('asn'): - misp_val = result['page']['asn'] - r.append({'types': 'AS', 'categories': ['External analysis'], 'values': misp_val, 'comment': f"{misp_comment} - ASN"}) - - if result['page'].get('asnname'): - misp_val = result['page']['asnname'] - r.append({'types': 'text', - 'categories': ['External analysis'], - 'values': misp_val, - 'comment': f"{misp_comment} - ASN name"}) - - if result['page'].get('tlsIssuer'): - misp_val = result['page']['tlsIssuer'] - r.append({'types': 'text', - 'categories': ['External analysis'], - 'values': misp_val, - 'comment': f"{misp_comment} - TLS Issuer"}) - - - if result['page'].get('title'): - misp_val = result['page']['title'] - r.append({'types': 'text', - 'categories': ['External analysis'], - 'values': misp_val, - 'comment': f"{misp_comment} - Page title"}) - - if result['page'].get('server'): - misp_val = result['page']['server'] - r.append({'types': 'text', - 'categories': ['External analysis'], - 'values': misp_val, - 'comment': f"{misp_comment} - Server"}) - - if result.get('stats'): - if result['stats'].get('malicious'): - log.debug('There is something in results > stats > malicious') + {"error": "Domain could not be resolved: {}".format(request["response"]["failed"]["errorText"])} + ) + + if result.get("page"): + if result["page"].get("domain"): + misp_val = result["page"]["domain"] + r.append( + { + "types": "domain", + "categories": ["Network activity"], + "values": misp_val, + "comment": f"{misp_comment} - Domain", + } + ) + + if result["page"].get("ip"): + misp_val = result["page"]["ip"] + r.append( + { + "types": "ip-dst", + "categories": ["Network activity"], + "values": misp_val, + "comment": f"{misp_comment} - IP", + } + ) + + if result["page"].get("ptr"): + misp_val = result["page"]["ptr"] + r.append( + { + "types": "hostname", + "categories": ["Network activity"], + "values": misp_val, + "comment": f"{misp_comment} - PTR", + } + ) + + if result["page"].get("country"): + misp_val = "country: " + result["page"]["country"] + if result["page"].get("city"): + misp_val += ", city: " + result["page"]["city"] + r.append( + { + "types": "text", + "categories": ["External analysis"], + "values": misp_val, + "comment": f"{misp_comment} - Country/City", + } + ) + + if result["page"].get("asn"): + misp_val = result["page"]["asn"] + r.append( + { + "types": "AS", + "categories": ["External analysis"], + "values": misp_val, + "comment": f"{misp_comment} - ASN", + } + ) + + if result["page"].get("asnname"): + misp_val = result["page"]["asnname"] + r.append( + { + "types": "text", + "categories": ["External analysis"], + "values": misp_val, + "comment": f"{misp_comment} - ASN name", + } + ) + + if result["page"].get("tlsIssuer"): + misp_val = result["page"]["tlsIssuer"] + r.append( + { + "types": "text", + "categories": ["External analysis"], + "values": misp_val, + "comment": f"{misp_comment} - TLS Issuer", + } + ) + + if result["page"].get("title"): + misp_val = result["page"]["title"] + r.append( + { + "types": "text", + "categories": ["External analysis"], + "values": misp_val, + "comment": f"{misp_comment} - Page title", + } + ) + + if result["page"].get("server"): + misp_val = result["page"]["server"] + r.append( + { + "types": "text", + "categories": ["External analysis"], + "values": misp_val, + "comment": f"{misp_comment} - Server", + } + ) + + if result.get("stats"): + if result["stats"].get("malicious"): + log.debug("There is something in results > stats > malicious") threat_list = set() - if 'matches' in result['meta']['processors']['gsb']['data']: - for item in result['meta']['processors']['gsb']['data']['matches']: - if item['threatType']: - threat_list.add(item['threatType']) + if "matches" in result["meta"]["processors"]["gsb"]["data"]: + for item in result["meta"]["processors"]["gsb"]["data"]["matches"]: + if item["threatType"]: + threat_list.add(item["threatType"]) - threat_list = ', '.join(threat_list) - log.debug('threat_list values are: \'' + threat_list + '\'') + threat_list = ", ".join(threat_list) + log.debug("threat_list values are: '" + threat_list + "'") if threat_list: - misp_val = '{} threat(s) detected'.format(threat_list) - r.append({'types': 'text', - 'categories': ['External analysis'], - 'values': misp_val, - 'comment': misp_comment}) - - if result.get('lists'): - if result['lists'].get('urls'): - for url in result['lists']['urls']: + misp_val = "{} threat(s) detected".format(threat_list) + r.append( + { + "types": "text", + "categories": ["External analysis"], + "values": misp_val, + "comment": misp_comment, + } + ) + + if result.get("lists"): + if result["lists"].get("urls"): + for url in result["lists"]["urls"]: url = url.lower() - if 'office' in url: + if "office" in url: misp_val = "Possible Office-themed phishing" - elif 'o365' in url or '0365' in url: + elif "o365" in url or "0365" in url: misp_val = "Possible O365-themed phishing" - elif 'microsoft' in url: + elif "microsoft" in url: misp_val = "Possible Microsoft-themed phishing" - elif 'paypal' in url: + elif "paypal" in url: misp_val = "Possible PayPal-themed phishing" - elif 'onedrive' in url: + elif "onedrive" in url: misp_val = "Possible OneDrive-themed phishing" - elif 'docusign' in url: + elif "docusign" in url: misp_val = "Possible DocuSign-themed phishing" - r.append({'types': 'text', - 'categories': ['External analysis'], - 'values': misp_val, - 'comment': misp_comment}) - - if result.get('task'): - if result['task'].get('reportURL'): - misp_val = result['task']['reportURL'] - r.append({'types': 'link', - 'categories': ['External analysis'], - 'values': misp_val, - 'comment': misp_comment}) - - if result['task'].get('screenshotURL'): - image_url = result['task']['screenshotURL'] - r.append({'types': 'link', - 'categories': ['External analysis'], - 'values': image_url, - 'comment': misp_comment}) + r.append( + { + "types": "text", + "categories": ["External analysis"], + "values": misp_val, + "comment": misp_comment, + } + ) + + if result.get("task"): + if result["task"].get("reportURL"): + misp_val = result["task"]["reportURL"] + r.append( + { + "types": "link", + "categories": ["External analysis"], + "values": misp_val, + "comment": misp_comment, + } + ) + + if result["task"].get("screenshotURL"): + image_url = result["task"]["screenshotURL"] + r.append( + { + "types": "link", + "categories": ["External analysis"], + "values": image_url, + "comment": misp_comment, + } + ) # ## TO DO ### # ## Add ability to add an in-line screenshot of the target website into an attribute # screenshot = requests.get(image_url).content @@ -223,48 +286,47 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo -class urlscanAPI(): +class urlscanAPI: def __init__(self, apikey=None, uuid=None): self.key = apikey self.uuid = uuid def request(self, query): - log.debug('From request function with the parameter: ' + query) - payload = {'url': query} - headers = {'API-Key': self.key, - 'Content-Type': "application/json", - 'Cache-Control': "no-cache"} + log.debug("From request function with the parameter: " + query) + payload = {"url": query} + headers = { + "API-Key": self.key, + "Content-Type": "application/json", + "Cache-Control": "no-cache", + } # Troubleshooting problems with initial search request - log.debug('PAYLOAD: ' + json.dumps(payload)) - log.debug('HEADERS: ' + json.dumps(headers)) + log.debug("PAYLOAD: " + json.dumps(payload)) + log.debug("HEADERS: " + json.dumps(headers)) search_url_string = "https://urlscan.io/api/v1/scan/" - response = requests.request("POST", - search_url_string, - data=json.dumps(payload), - headers=headers) + response = requests.request("POST", search_url_string, data=json.dumps(payload), headers=headers) # HTTP 400 - Bad Request if response.status_code == 400: - raise Exception('HTTP Error 400 - Bad Request') + raise Exception("HTTP Error 400 - Bad Request") # HTTP 404 - Not found if response.status_code == 404: - raise Exception('HTTP Error 404 - These are not the droids you\'re looking for') + raise Exception("HTTP Error 404 - These are not the droids you're looking for") # Any other status code if response.status_code != 200: - raise Exception('HTTP Error ' + str(response.status_code)) + raise Exception("HTTP Error " + str(response.status_code)) if response.text: response = json.loads(response.content.decode("utf-8")) time.sleep(3) - self.uuid = response['uuid'] + self.uuid = response["uuid"] # Strings for to check for errors on the results page # Null response string for any unavailable resources @@ -275,26 +337,28 @@ def request(self, query): normal_response_string = '"status": 200' results_url_string = "https://urlscan.io/api/v1/result/" + self.uuid - log.debug('Results URL: ' + results_url_string) + log.debug("Results URL: " + results_url_string) # Need to wait for results to process and check if they are valid tries = 10 while tries >= 0: results = requests.request("GET", results_url_string) - log.debug('Made a GET request') + log.debug("Made a GET request") results = results.content.decode("utf-8") # checking if there is a 404 status code and no available resources - if null_response_string in results and \ - redirect_string not in results and \ - normal_response_string not in results: - log.debug('Results not processed. Please check again later.') + if ( + null_response_string in results + and redirect_string not in results + and normal_response_string not in results + ): + log.debug("Results not processed. Please check again later.") time.sleep(3) tries -= 1 else: return json.loads(results) - raise Exception('Results contained a 404 status error and could not be processed.') + raise Exception("Results contained a 404 status error and could not be processed.") def search_url(self, query): - log.debug('From search_url with parameter: ' + query) + log.debug("From search_url with parameter: " + query) return self.request(query) diff --git a/misp_modules/modules/expansion/variotdbs.py b/misp_modules/modules/expansion/variotdbs.py index 8526949b6..087594e43 100644 --- a/misp_modules/modules/expansion/variotdbs.py +++ b/misp_modules/modules/expansion/variotdbs.py @@ -1,45 +1,48 @@ import json + import requests -from . import check_input_attribute, standard_error_message -from ._vulnerability_parser.vulnerability_parser import ( - VulnerabilityMapping, VulnerabilityParser) from pymisp import MISPObject -misperrors = {'error': 'Error'} -mispattributes = {'input': ['vulnerability'], 'format': 'misp_standard'} +from . import check_input_attribute, standard_error_message +from ._vulnerability_parser.vulnerability_parser import VulnerabilityMapping, VulnerabilityParser + +misperrors = {"error": "Error"} +mispattributes = {"input": ["vulnerability"], "format": "misp_standard"} moduleinfo = { - 'version': '1', - 'author': 'Christian Studer', - 'description': 'An expansion module to query the VARIoT db API for more information about a vulnerability.', - 'module-type': ['expansion', 'hover'], - 'name': 'VARIoT db Lookup', - 'logo': 'variot.png', - 'requirements': ['A VARIoT db API key (if you do not want to be limited to 100 queries / day)'], - 'features': 'The module takes a vulnerability attribute as input and queries que VARIoT db API to gather additional information.\n\nThe `vuln` endpoint is queried first to look for additional information about the vulnerability itself.\n\nThe `exploits` endpoint is also queried then to look for the information of the potential related exploits, which are parsed and added to the results using the `exploit` object template.', - 'references': ['https://www.variotdbs.pl/'], - 'input': 'Vulnerability attribute.', - 'output': 'Additional information about the vulnerability, as it is stored on the VARIoT db, about the vulnerability itself, and the potential related exploits.', + "version": "1", + "author": "Christian Studer", + "description": "An expansion module to query the VARIoT db API for more information about a vulnerability.", + "module-type": ["expansion", "hover"], + "name": "VARIoT db Lookup", + "logo": "variot.png", + "requirements": ["A VARIoT db API key (if you do not want to be limited to 100 queries / day)"], + "features": ( + "The module takes a vulnerability attribute as input and queries que VARIoT db API to gather additional" + " information.\n\nThe `vuln` endpoint is queried first to look for additional information about the" + " vulnerability itself.\n\nThe `exploits` endpoint is also queried then to look for the information of the" + " potential related exploits, which are parsed and added to the results using the `exploit` object template." + ), + "references": ["https://www.variotdbs.pl/"], + "input": "Vulnerability attribute.", + "output": ( + "Additional information about the vulnerability, as it is stored on the VARIoT db, about the vulnerability" + " itself, and the potential related exploits." + ), } -moduleconfig = ['API_key'] -variotdbs_url = 'https://www.variotdbs.pl/api' +moduleconfig = ["API_key"] +variotdbs_url = "https://www.variotdbs.pl/api" class VariotMapping(VulnerabilityMapping): __exploit_mapping = { - 'credits': 'credit', - 'description': 'description', - 'exploit': 'exploit', - 'title': 'title' + "credits": "credit", + "description": "description", + "exploit": "exploit", + "title": "title", } __exploit_multiple_mapping = { - 'cve': { - 'feature': 'cve_id', - 'relation': 'cve-id' - }, - 'references': { - 'feature': 'url', - 'relation': 'reference' - } + "cve": {"feature": "cve_id", "relation": "cve-id"}, + "references": {"feature": "url", "relation": "reference"}, } @classmethod @@ -62,20 +65,16 @@ def mapping(self) -> VulnerabilityMapping: def parse_exploit_information(self, query_results): for exploit in query_results: - exploit_object = MISPObject('exploit') - exploit_object.add_attribute('exploitdb-id', exploit['edb_id']) + exploit_object = MISPObject("exploit") + exploit_object.add_attribute("exploitdb-id", exploit["edb_id"]) for field, relation in self.mapping.exploit_mapping().items(): if exploit.get(field): - exploit_object.add_attribute( - relation, exploit[field]['data'] - ) + exploit_object.add_attribute(relation, exploit[field]["data"]) for field, relation in self.mapping.exploit_multiple_mapping().items(): if exploit.get(field): - for value in exploit[field]['data']: - exploit_object.add_attribute( - relation['relation'], value[relation['feature']] - ) - exploit_object.add_reference(self.misp_attribute.uuid, 'related-to') + for value in exploit[field]["data"]: + exploit_object.add_attribute(relation["relation"], value[relation["feature"]]) + exploit_object.add_reference(self.misp_attribute.uuid, "related-to") self.misp_event.add_object(exploit_object) @@ -83,14 +82,14 @@ def handler(q=False): if q is False: return False request = json.loads(q) - if not request.get('attribute') or not check_input_attribute(request['attribute']): - return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} - attribute = request['attribute'] - if attribute.get('type') != 'vulnerability': - return {'error': 'Vulnerability id missing.'} - headers = {'Content-Type': 'application/json'} - if request.get('config', {}).get('API_key'): - headers['Authorization'] = f"Token {request['config']['API_key']}" + if not request.get("attribute") or not check_input_attribute(request["attribute"]): + return {"error": f"{standard_error_message}, which should contain at least a type, a value and an uuid."} + attribute = request["attribute"] + if attribute.get("type") != "vulnerability": + return {"error": "Vulnerability id missing."} + headers = {"Content-Type": "application/json"} + if request.get("config", {}).get("API_key"): + headers["Authorization"] = f"Token {request['config']['API_key']}" empty = True parser = VariotdbsParser(attribute) r = requests.get(f"{variotdbs_url}/vuln/{attribute['value']}/", headers=headers) @@ -100,27 +99,27 @@ def handler(q=False): parser._parse_variot_description(vulnerability_results) empty = False else: - if r.reason != 'Not Found': - return {'error': 'Error while querying the variotdbs API.'} + if r.reason != "Not Found": + return {"error": "Error while querying the variotdbs API."} r = requests.get(f"{variotdbs_url}/exploits/?cve={attribute['value']}", headers=headers) if r.status_code == 200: exploit_results = r.json() if exploit_results: - parser.parse_exploit_information(exploit_results['results']) + parser.parse_exploit_information(exploit_results["results"]) empty = False - if exploit_results['next'] is not None: - while(1): - exploit_results = requests.get(exploit_results['next'], headers=headers) + if exploit_results["next"] is not None: + while 1: + exploit_results = requests.get(exploit_results["next"], headers=headers) if exploit_results.status_code != 200: break exploit_results = exploit_results.json() - parser.parse_exploit_information(exploit_results['results']) - if exploit_results['next'] is None: + parser.parse_exploit_information(exploit_results["results"]) + if exploit_results["next"] is None: break else: - return {'error': 'Error while querying the variotdbs API.'} + return {"error": "Error while querying the variotdbs API."} if empty: - return {'error': 'Empty results'} + return {"error": "Empty results"} return parser.get_results() @@ -129,5 +128,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/virustotal.py b/misp_modules/modules/expansion/virustotal.py index 828efb607..5e3d4f691 100644 --- a/misp_modules/modules/expansion/virustotal.py +++ b/misp_modules/modules/expansion/virustotal.py @@ -1,29 +1,65 @@ from urllib.parse import urlparse + import vt -from . import check_input_attribute, standard_error_message from pymisp import MISPAttribute, MISPEvent, MISPObject -misperrors = {'error': 'Error'} -mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst", "md5", "sha1", "sha256", "url", "ip-src|port", "ip-dst|port"], - 'format': 'misp_standard'} +from . import check_input_attribute, standard_error_message + +misperrors = {"error": "Error"} +mispattributes = { + "input": [ + "hostname", + "domain", + "ip-src", + "ip-dst", + "md5", + "sha1", + "sha256", + "url", + "ip-src|port", + "ip-dst|port", + ], + "format": "misp_standard", +} # possible module-types: 'expansion', 'hover' or both moduleinfo = { - 'version': '6', - 'author': 'Hannah Ward', - 'description': 'Enrich observables with the VirusTotal v3 API', - 'module-type': ['expansion'], - 'name': 'VirusTotal v3 Lookup', - 'logo': 'virustotal.png', - 'requirements': ['An access to the VirusTotal API (apikey), with a high request rate limit.'], - 'features': 'New format of modules able to return attributes and objects.\n\nA module to take a MISP attribute as input and query the VirusTotal API to get additional data about it.\n\nCompared to the [standard VirusTotal expansion module](https://github.com/MISP/misp-modules/blob/main/misp_modules/modules/expansion/virustotal_public.py), this module is made for advanced parsing of VirusTotal report, with a recursive analysis of the elements found after the first request.\n\nThus, it requires a higher request rate limit to avoid the API to return a 204 error (Request rate limit exceeded), and the data parsed from the different requests are returned as MISP attributes and objects, with the corresponding relations between each one of them.', - 'references': ['https://www.virustotal.com/', 'https://docs.virustotal.com/reference/overview'], - 'input': 'A domain, hash (md5, sha1, sha256 or sha512), hostname or IP address attribute.', - 'output': 'MISP attributes and objects resulting from the parsing of the VirusTotal report concerning the input attribute.', + "version": "6", + "author": "Hannah Ward", + "description": "Enrich observables with the VirusTotal v3 API", + "module-type": ["expansion"], + "name": "VirusTotal v3 Lookup", + "logo": "virustotal.png", + "requirements": ["An access to the VirusTotal API (apikey), with a high request rate limit."], + "features": ( + "New format of modules able to return attributes and objects.\n\nA module to take a MISP attribute as input and" + " query the VirusTotal API to get additional data about it.\n\nCompared to the [standard VirusTotal expansion" + " module](https://github.com/MISP/misp-modules/blob/main/misp_modules/modules/expansion/virustotal_public.py)," + " this module is made for advanced parsing of VirusTotal report, with a recursive analysis of the elements" + " found after the first request.\n\nThus, it requires a higher request rate limit to avoid the API to return a" + " 204 error (Request rate limit exceeded), and the data parsed from the different requests are returned as MISP" + " attributes and objects, with the corresponding relations between each one of them." + ), + "references": [ + "https://www.virustotal.com/", + "https://docs.virustotal.com/reference/overview", + ], + "input": "A domain, hash (md5, sha1, sha256 or sha512), hostname or IP address attribute.", + "output": ( + "MISP attributes and objects resulting from the parsing of the VirusTotal report concerning the input" + " attribute." + ), } # config fields that your code expects from the site admin -moduleconfig = ["apikey", "event_limit", 'proxy_host', 'proxy_port', 'proxy_username', 'proxy_password'] +moduleconfig = [ + "apikey", + "event_limit", + "proxy_host", + "proxy_port", + "proxy_username", + "proxy_password", +] DEFAULT_RESULTS_LIMIT = 10 @@ -36,19 +72,26 @@ def __init__(self, client: vt.Client, limit: int) -> None: self.misp_event = MISPEvent() self.attribute = MISPAttribute() self.parsed_objects = {} - self.input_types_mapping = {'ip-src': self.parse_ip, 'ip-dst': self.parse_ip, - 'domain': self.parse_domain, 'hostname': self.parse_domain, - 'md5': self.parse_hash, 'sha1': self.parse_hash, - 'sha256': self.parse_hash, 'url': self.parse_url, - 'ip-src|port': self.parse_ip_port, 'ip-dst|port': self.parse_ip_port} + self.input_types_mapping = { + "ip-src": self.parse_ip, + "ip-dst": self.parse_ip, + "domain": self.parse_domain, + "hostname": self.parse_domain, + "md5": self.parse_hash, + "sha1": self.parse_hash, + "sha256": self.parse_hash, + "url": self.parse_url, + "ip-src|port": self.parse_ip_port, + "ip-dst|port": self.parse_ip_port, + } self.proxies = None @staticmethod def get_total_analysis(analysis: dict, known_distributors: dict = None) -> int: if not analysis: return 0 - count = sum([analysis['undetected'], analysis['suspicious'], analysis['harmless']]) - return count if known_distributors else count + analysis['malicious'] + count = sum([analysis["undetected"], analysis["suspicious"], analysis["harmless"]]) + return count if known_distributors else count + analysis["malicious"] def query_api(self, attribute: dict) -> None: self.attribute.from_dict(**attribute) @@ -56,22 +99,27 @@ def query_api(self, attribute: dict) -> None: def get_result(self) -> dict: event = self.misp_event.to_dict() - results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} - return {'results': results} + results = {key: event[key] for key in ("Attribute", "Object") if (key in event and event[key])} + return {"results": results} def add_vt_report(self, report: vt.Object) -> str: - analysis = report.get('last_analysis_stats') - total = self.get_total_analysis(analysis, report.get('known_distributors')) - if report.type == 'ip_address': - rtype = 'ip-address' + analysis = report.get("last_analysis_stats") + total = self.get_total_analysis(analysis, report.get("known_distributors")) + if report.type == "ip_address": + rtype = "ip-address" else: rtype = report.type - permalink = f'https://www.virustotal.com/gui/{rtype}/{report.id}' - - vt_object = MISPObject('virustotal-report') - vt_object.add_attribute('permalink', type='link', value=permalink) - detection_ratio = f"{analysis['malicious']}/{total}" if analysis else '-/-' - vt_object.add_attribute('detection-ratio', type='text', value=detection_ratio, disable_correlation=True) + permalink = f"https://www.virustotal.com/gui/{rtype}/{report.id}" + + vt_object = MISPObject("virustotal-report") + vt_object.add_attribute("permalink", type="link", value=permalink) + detection_ratio = f"{analysis['malicious']}/{total}" if analysis else "-/-" + vt_object.add_attribute( + "detection-ratio", + type="text", + value=detection_ratio, + disable_correlation=True, + ) self.misp_event.add_object(**vt_object) return vt_object.uuid @@ -79,23 +127,28 @@ def create_misp_object(self, report: vt.Object) -> MISPObject: misp_object = None vt_uuid = self.add_vt_report(report) - if report.type == 'file': - misp_object = MISPObject('file') - for hash_type in ('md5', 'sha1', 'sha256', 'tlsh', - 'vhash', 'ssdeep', 'imphash'): - misp_object.add_attribute(hash_type, - **{'type': hash_type, - 'value': report.get(hash_type)}) - elif report.type == 'domain': - misp_object = MISPObject('domain-ip') - misp_object.add_attribute('domain', type='domain', value=report.id) - elif report.type == 'ip_address': - misp_object = MISPObject('domain-ip') - misp_object.add_attribute('ip', type='ip-dst', value=report.id) - elif report.type == 'url': - misp_object = MISPObject('url') - misp_object.add_attribute('url', type='url', value=report.id) - misp_object.add_reference(vt_uuid, 'analyzed-with') + if report.type == "file": + misp_object = MISPObject("file") + for hash_type in ( + "md5", + "sha1", + "sha256", + "tlsh", + "vhash", + "ssdeep", + "imphash", + ): + misp_object.add_attribute(hash_type, **{"type": hash_type, "value": report.get(hash_type)}) + elif report.type == "domain": + misp_object = MISPObject("domain-ip") + misp_object.add_attribute("domain", type="domain", value=report.id) + elif report.type == "ip_address": + misp_object = MISPObject("domain-ip") + misp_object.add_attribute("ip", type="ip-dst", value=report.id) + elif report.type == "url": + misp_object = MISPObject("url") + misp_object.add_attribute("url", type="url", value=report.id) + misp_object.add_reference(vt_uuid, "analyzed-with") return misp_object ################################################################################ @@ -103,71 +156,76 @@ def create_misp_object(self, report: vt.Object) -> MISPObject: ################################################################################ def parse_domain(self, domain: str) -> str: - domain_report = self.client.get_object(f'/domains/{domain}') + domain_report = self.client.get_object(f"/domains/{domain}") # DOMAIN domain_object = self.create_misp_object(domain_report) # WHOIS if domain_report.whois: - whois_object = MISPObject('whois') - whois_object.add_attribute('text', type='text', value=domain_report.whois) + whois_object = MISPObject("whois") + whois_object.add_attribute("text", type="text", value=domain_report.whois) self.misp_event.add_object(**whois_object) # SIBLINGS AND SUBDOMAINS - for relationship_name, misp_name in [('siblings', 'sibling-of'), ('subdomains', 'subdomain')]: - rel_iterator = self.client.iterator(f'/domains/{domain_report.id}/{relationship_name}', limit=self.limit) + for relationship_name, misp_name in [ + ("siblings", "sibling-of"), + ("subdomains", "subdomain"), + ]: + rel_iterator = self.client.iterator(f"/domains/{domain_report.id}/{relationship_name}", limit=self.limit) for item in rel_iterator: attr = MISPAttribute() - attr.from_dict(**dict(type='domain', value=item.id)) + attr.from_dict(**dict(type="domain", value=item.id)) self.misp_event.add_attribute(**attr) domain_object.add_reference(attr.uuid, misp_name) # RESOLUTIONS - resolutions_iterator = self.client.iterator(f'/domains/{domain_report.id}/resolutions', limit=self.limit) + resolutions_iterator = self.client.iterator(f"/domains/{domain_report.id}/resolutions", limit=self.limit) for resolution in resolutions_iterator: - domain_object.add_attribute('ip', type='ip-dst', value=resolution.ip_address) + domain_object.add_attribute("ip", type="ip-dst", value=resolution.ip_address) # COMMUNICATING, DOWNLOADED AND REFERRER FILES for relationship_name, misp_name in [ - ('communicating_files', 'communicates-with'), - ('downloaded_files', 'downloaded-from'), - ('referrer_files', 'referring') + ("communicating_files", "communicates-with"), + ("downloaded_files", "downloaded-from"), + ("referrer_files", "referring"), ]: - files_iterator = self.client.iterator(f'/domains/{domain_report.id}/{relationship_name}', limit=self.limit) + files_iterator = self.client.iterator(f"/domains/{domain_report.id}/{relationship_name}", limit=self.limit) for file in files_iterator: file_object = self.create_misp_object(file) file_object.add_reference(domain_object.uuid, misp_name) self.misp_event.add_object(**file_object) # URLS - urls_iterator = self.client.iterator(f'/domains/{domain_report.id}/urls', limit=self.limit) + urls_iterator = self.client.iterator(f"/domains/{domain_report.id}/urls", limit=self.limit) for url in urls_iterator: url_object = self.create_misp_object(url) - url_object.add_reference(domain_object.uuid, 'hosted-in') + url_object.add_reference(domain_object.uuid, "hosted-in") self.misp_event.add_object(**url_object) self.misp_event.add_object(**domain_object) return domain_object.uuid def parse_hash(self, file_hash: str) -> str: - file_report = self.client.get_object(f'/files/{file_hash}') + file_report = self.client.get_object(f"/files/{file_hash}") file_object = self.create_misp_object(file_report) # ITW URLS - urls_iterator = self.client.iterator(f'/files/{file_report.id}/itw_urls', limit=self.limit) + urls_iterator = self.client.iterator(f"/files/{file_report.id}/itw_urls", limit=self.limit) for url in urls_iterator: url_object = self.create_misp_object(url) - url_object.add_reference(file_object.uuid, 'downloaded') + url_object.add_reference(file_object.uuid, "downloaded") self.misp_event.add_object(**url_object) # COMMUNICATING, DOWNLOADED AND REFERRER FILES for relationship_name, misp_name in [ - ('contacted_urls', 'communicates-with'), - ('contacted_domains', 'communicates-with'), - ('contacted_ips', 'communicates-with') + ("contacted_urls", "communicates-with"), + ("contacted_domains", "communicates-with"), + ("contacted_ips", "communicates-with"), ]: - related_files_iterator = self.client.iterator(f'/files/{file_report.id}/{relationship_name}', limit=self.limit) + related_files_iterator = self.client.iterator( + f"/files/{file_report.id}/{relationship_name}", limit=self.limit + ) for related_file in related_files_iterator: related_file_object = self.create_misp_object(related_file) related_file_object.add_reference(file_object.uuid, misp_name) @@ -175,33 +233,34 @@ def parse_hash(self, file_hash: str) -> str: self.misp_event.add_object(**file_object) return file_object.uuid + def parse_ip_port(self, ipport: str) -> str: - ip = ipport.split('|')[0] + ip = ipport.split("|")[0] self.parse_ip(ip) def parse_ip(self, ip: str) -> str: - ip_report = self.client.get_object(f'/ip_addresses/{ip}') + ip_report = self.client.get_object(f"/ip_addresses/{ip}") # IP ip_object = self.create_misp_object(ip_report) # ASN - asn_object = MISPObject('asn') - asn_object.add_attribute('asn', type='AS', value=ip_report.asn) - asn_object.add_attribute('subnet-announced', type='ip-src', value=ip_report.network) - asn_object.add_attribute('country', type='text', value=ip_report.country) + asn_object = MISPObject("asn") + asn_object.add_attribute("asn", type="AS", value=ip_report.asn) + asn_object.add_attribute("subnet-announced", type="ip-src", value=ip_report.network) + asn_object.add_attribute("country", type="text", value=ip_report.country) self.misp_event.add_object(**asn_object) # RESOLUTIONS - resolutions_iterator = self.client.iterator(f'/ip_addresses/{ip_report.id}/resolutions', limit=self.limit) + resolutions_iterator = self.client.iterator(f"/ip_addresses/{ip_report.id}/resolutions", limit=self.limit) for resolution in resolutions_iterator: - ip_object.add_attribute('domain', type='domain', value=resolution.host_name) + ip_object.add_attribute("domain", type="domain", value=resolution.host_name) # URLS - urls_iterator = self.client.iterator(f'/ip_addresses/{ip_report.id}/urls', limit=self.limit) + urls_iterator = self.client.iterator(f"/ip_addresses/{ip_report.id}/urls", limit=self.limit) for url in urls_iterator: url_object = self.create_misp_object(url) - url_object.add_reference(ip_object.uuid, 'hosted-in') + url_object.add_reference(ip_object.uuid, "hosted-in") self.misp_event.add_object(**url_object) self.misp_event.add_object(**ip_object) @@ -209,16 +268,16 @@ def parse_ip(self, ip: str) -> str: def parse_url(self, url: str) -> str: url_id = vt.url_id(url) - url_report = self.client.get_object(f'/urls/{url_id}') + url_report = self.client.get_object(f"/urls/{url_id}") url_object = self.create_misp_object(url_report) # COMMUNICATING, DOWNLOADED AND REFERRER FILES for relationship_name, misp_name in [ - ('communicating_files', 'communicates-with'), - ('downloaded_files', 'downloaded-from'), - ('referrer_files', 'referring') + ("communicating_files", "communicates-with"), + ("downloaded_files", "downloaded-from"), + ("referrer_files", "referring"), ]: - files_iterator = self.client.iterator(f'/urls/{url_report.id}/{relationship_name}', limit=self.limit) + files_iterator = self.client.iterator(f"/urls/{url_report.id}/{relationship_name}", limit=self.limit) for file in files_iterator: file_object = self.create_misp_object(file) file_object.add_reference(url_object.uuid, misp_name) @@ -232,74 +291,75 @@ def get_proxy_settings(config: dict) -> dict: """Returns proxy settings in the requests format. If no proxy settings are set, return None.""" proxies = None - host = config.get('proxy_host') - port = config.get('proxy_port') - username = config.get('proxy_username') - password = config.get('proxy_password') + host = config.get("proxy_host") + port = config.get("proxy_port") + username = config.get("proxy_username") + password = config.get("proxy_password") if host: if not port: - misperrors['error'] = 'The virustotal_proxy_host config is set, ' \ - 'please also set the virustotal_proxy_port.' + misperrors["error"] = "The virustotal_proxy_host config is set, please also set the virustotal_proxy_port." raise KeyError parsed = urlparse(host) - if 'http' in parsed.scheme: - scheme = 'http' + if "http" in parsed.scheme: + scheme = "http" else: scheme = parsed.scheme netloc = parsed.netloc - host = f'{netloc}:{port}' + host = f"{netloc}:{port}" if username: if not password: - misperrors['error'] = 'The virustotal_proxy_username config is set, ' \ - 'please also set the virustotal_proxy_password.' + misperrors["error"] = ( + "The virustotal_proxy_username config is set, please also set the virustotal_proxy_password." + ) raise KeyError - auth = f'{username}:{password}' - host = auth + '@' + host + auth = f"{username}:{password}" + host = auth + "@" + host - proxies = { - 'http': f'{scheme}://{host}', - 'https': f'{scheme}://{host}' - } + proxies = {"http": f"{scheme}://{host}", "https": f"{scheme}://{host}"} return proxies def parse_error(status_code: int) -> str: - status_mapping = {204: 'VirusTotal request rate limit exceeded.', - 400: 'Incorrect request, please check the arguments.', - 403: 'You don\'t have enough privileges to make the request.'} + status_mapping = { + 204: "VirusTotal request rate limit exceeded.", + 400: "Incorrect request, please check the arguments.", + 403: "You don't have enough privileges to make the request.", + } if status_code in status_mapping: return status_mapping[status_code] return "VirusTotal may not be accessible." def dict_handler(request: dict): - if not request.get('config') or not request['config'].get('apikey'): - misperrors['error'] = 'A VirusTotal api key is required for this module.' + if not request.get("config") or not request["config"].get("apikey"): + misperrors["error"] = "A VirusTotal api key is required for this module." return misperrors - if not request.get('attribute') or not check_input_attribute(request['attribute']): - return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} - if request['attribute']['type'] not in mispattributes['input']: - return {'error': 'Unsupported attribute type.'} + if not request.get("attribute") or not check_input_attribute(request["attribute"]): + return {"error": f"{standard_error_message}, which should contain at least a type, a value and an uuid."} + if request["attribute"]["type"] not in mispattributes["input"]: + return {"error": "Unsupported attribute type."} - event_limit = request['config'].get('event_limit') - attribute = request['attribute'] - proxy_settings = get_proxy_settings(request.get('config')) + event_limit = request["config"].get("event_limit") + attribute = request["attribute"] + proxy_settings = get_proxy_settings(request.get("config")) try: - client = vt.Client(request['config']['apikey'], - headers={ - 'x-tool': 'MISPModuleVirusTotalExpansion', - }, - proxy=proxy_settings['http'] if proxy_settings else None) + client = vt.Client( + request["config"]["apikey"], + headers={ + "x-tool": "MISPModuleVirusTotalExpansion", + }, + proxy=proxy_settings["http"] if proxy_settings else None, + ) parser = VirusTotalParser(client, int(event_limit) if event_limit else None) parser.query_api(attribute) except vt.APIError as ex: - if ex.code == 'ForbiddenError': - misperrors['error'] = 'ForbiddenError' + if ex.code == "ForbiddenError": + misperrors["error"] = "ForbiddenError" else: - misperrors['error'] = ex.message + misperrors["error"] = ex.message return misperrors return parser.get_result() @@ -310,5 +370,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/virustotal_public.py b/misp_modules/modules/expansion/virustotal_public.py index d8de28630..95cd70c88 100644 --- a/misp_modules/modules/expansion/virustotal_public.py +++ b/misp_modules/modules/expansion/virustotal_public.py @@ -1,30 +1,53 @@ import json import logging -import vt -from . import check_input_attribute, standard_error_message from urllib.parse import urlparse + +import vt from pymisp import MISPAttribute, MISPEvent, MISPObject -misperrors = {'error': 'Error'} -mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst", "md5", "sha1", "sha256", "url"], - 'format': 'misp_standard'} +from . import check_input_attribute, standard_error_message + +misperrors = {"error": "Error"} +mispattributes = { + "input": ["hostname", "domain", "ip-src", "ip-dst", "md5", "sha1", "sha256", "url"], + "format": "misp_standard", +} moduleinfo = { - 'version': '2', - 'author': 'Christian Studer', - 'description': 'Enrich observables with the VirusTotal v3 public API', - 'module-type': ['expansion', 'hover'], - 'name': 'VirusTotal Public API Lookup', - 'logo': 'virustotal.png', - 'requirements': ['An access to the VirusTotal API (apikey)'], - 'features': 'New format of modules able to return attributes and objects.\n\nA module to take a MISP attribute as input and query the VirusTotal API to get additional data about it.\n\nCompared to the [more advanced VirusTotal expansion module](https://github.com/MISP/misp-modules/blob/main/misp_modules/modules/expansion/virustotal.py), this module is made for VirusTotal users who have a low request rate limit.\n\nThus, it only queries the API once and returns the results that is parsed into MISP attributes and objects.', - 'references': ['https://www.virustotal.com', 'https://docs.virustotal.com/reference/overview'], - 'input': 'A domain, hostname, ip, url or hash (md5, sha1, sha256 or sha512) attribute.', - 'output': 'MISP attributes and objects resulting from the parsing of the VirusTotal report concerning the input attribute.', + "version": "2", + "author": "Christian Studer", + "description": "Enrich observables with the VirusTotal v3 public API", + "module-type": ["expansion", "hover"], + "name": "VirusTotal Public API Lookup", + "logo": "virustotal.png", + "requirements": ["An access to the VirusTotal API (apikey)"], + "features": ( + "New format of modules able to return attributes and objects.\n\nA module to take a MISP attribute as input and" + " query the VirusTotal API to get additional data about it.\n\nCompared to the [more advanced VirusTotal" + " expansion" + " module](https://github.com/MISP/misp-modules/blob/main/misp_modules/modules/expansion/virustotal.py), this" + " module is made for VirusTotal users who have a low request rate limit.\n\nThus, it only queries the API once" + " and returns the results that is parsed into MISP attributes and objects." + ), + "references": [ + "https://www.virustotal.com", + "https://docs.virustotal.com/reference/overview", + ], + "input": "A domain, hostname, ip, url or hash (md5, sha1, sha256 or sha512) attribute.", + "output": ( + "MISP attributes and objects resulting from the parsing of the VirusTotal report concerning the input" + " attribute." + ), } -moduleconfig = ['apikey', 'proxy_host', 'proxy_port', 'proxy_username', 'proxy_password'] +moduleconfig = [ + "apikey", + "proxy_host", + "proxy_port", + "proxy_username", + "proxy_password", +] -LOGGER = logging.getLogger('virus_total_public') +LOGGER = logging.getLogger("virus_total_public") LOGGER.setLevel(logging.INFO) @@ -38,18 +61,24 @@ def __init__(self, client: vt.Client, limit: int) -> None: self.misp_event = MISPEvent() self.attribute = MISPAttribute() self.parsed_objects = {} - self.input_types_mapping = {'ip-src': self.parse_ip, 'ip-dst': self.parse_ip, - 'domain': self.parse_domain, 'hostname': self.parse_domain, - 'md5': self.parse_hash, 'sha1': self.parse_hash, - 'sha256': self.parse_hash, 'url': self.parse_url} + self.input_types_mapping = { + "ip-src": self.parse_ip, + "ip-dst": self.parse_ip, + "domain": self.parse_domain, + "hostname": self.parse_domain, + "md5": self.parse_hash, + "sha1": self.parse_hash, + "sha256": self.parse_hash, + "url": self.parse_url, + } self.proxies = None @staticmethod def get_total_analysis(analysis: dict, known_distributors: dict = None) -> int: if not analysis: return 0 - count = sum([analysis['undetected'], analysis['suspicious'], analysis['harmless']]) - return count if known_distributors else count + analysis['malicious'] + count = sum([analysis["undetected"], analysis["suspicious"], analysis["harmless"]]) + return count if known_distributors else count + analysis["malicious"] def query_api(self, attribute: dict) -> None: self.attribute.from_dict(**attribute) @@ -57,41 +86,57 @@ def query_api(self, attribute: dict) -> None: def get_result(self) -> dict: event = json.loads(self.misp_event.to_json()) - results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} - return {'results': results} + results = {key: event[key] for key in ("Attribute", "Object") if (key in event and event[key])} + return {"results": results} def add_vt_report(self, report: vt.Object) -> str: - analysis = report.get('last_analysis_stats') - total = self.get_total_analysis(analysis, report.get('known_distributors')) - permalink = f'https://www.virustotal.com/gui/{report.type}/{report.id}' - - vt_object = MISPObject('virustotal-report') - vt_object.add_attribute('permalink', type='link', value=permalink) - detection_ratio = f"{analysis['malicious']}/{total}" if analysis else '-/-' - vt_object.add_attribute('detection-ratio', type='text', value=detection_ratio, disable_correlation=True) + analysis = report.get("last_analysis_stats") + total = self.get_total_analysis(analysis, report.get("known_distributors")) + permalink = f"https://www.virustotal.com/gui/{report.type}/{report.id}" + + vt_object = MISPObject("virustotal-report") + vt_object.add_attribute("permalink", type="link", value=permalink) + detection_ratio = f"{analysis['malicious']}/{total}" if analysis else "-/-" + vt_object.add_attribute( + "detection-ratio", + type="text", + value=detection_ratio, + disable_correlation=True, + ) self.misp_event.add_object(**vt_object) return vt_object.uuid def create_misp_object(self, report: vt.Object) -> MISPObject: misp_object = None vt_uuid = self.add_vt_report(report) - if report.type == 'file': - misp_object = MISPObject('file') - for hash_type in ('md5', 'sha1', 'sha256', 'tlsh', - 'vhash', 'ssdeep', 'imphash'): - misp_object.add_attribute(**{'type': hash_type, - 'object_relation': hash_type, - 'value': report.get(hash_type)}) - elif report.type == 'domain': - misp_object = MISPObject('domain-ip') - misp_object.add_attribute('domain', type='domain', value=report.id) - elif report.type == 'ip_address': - misp_object = MISPObject('domain-ip') - misp_object.add_attribute('ip', type='ip-dst', value=report.id) - elif report.type == 'url': - misp_object = MISPObject('url') - misp_object.add_attribute('url', type='url', value=report.id) - misp_object.add_reference(vt_uuid, 'analyzed-with') + if report.type == "file": + misp_object = MISPObject("file") + for hash_type in ( + "md5", + "sha1", + "sha256", + "tlsh", + "vhash", + "ssdeep", + "imphash", + ): + misp_object.add_attribute( + **{ + "type": hash_type, + "object_relation": hash_type, + "value": report.get(hash_type), + } + ) + elif report.type == "domain": + misp_object = MISPObject("domain-ip") + misp_object.add_attribute("domain", type="domain", value=report.id) + elif report.type == "ip_address": + misp_object = MISPObject("domain-ip") + misp_object.add_attribute("ip", type="ip-dst", value=report.id) + elif report.type == "url": + misp_object = MISPObject("url") + misp_object.add_attribute("url", type="url", value=report.id) + misp_object.add_reference(vt_uuid, "analyzed-with") return misp_object ################################################################################ @@ -99,37 +144,40 @@ def create_misp_object(self, report: vt.Object) -> MISPObject: ################################################################################ def parse_domain(self, domain: str) -> str: - domain_report = self.client.get_object(f'/domains/{domain}') + domain_report = self.client.get_object(f"/domains/{domain}") # DOMAIN domain_object = self.create_misp_object(domain_report) # WHOIS if domain_report.whois: - whois_object = MISPObject('whois') - whois_object.add_attribute('text', type='text', value=domain_report.whois) + whois_object = MISPObject("whois") + whois_object.add_attribute("text", type="text", value=domain_report.whois) self.misp_event.add_object(**whois_object) # SIBLINGS AND SUBDOMAINS - for relationship_name, misp_name in [('siblings', 'sibling-of'), ('subdomains', 'subdomain')]: - rel_iterator = self.client.iterator(f'/domains/{domain_report.id}/{relationship_name}', limit=self.limit) + for relationship_name, misp_name in [ + ("siblings", "sibling-of"), + ("subdomains", "subdomain"), + ]: + rel_iterator = self.client.iterator(f"/domains/{domain_report.id}/{relationship_name}", limit=self.limit) for item in rel_iterator: attr = MISPAttribute() - attr.from_dict(**dict(type='domain', value=item.id)) + attr.from_dict(**dict(type="domain", value=item.id)) self.misp_event.add_attribute(**attr) domain_object.add_reference(attr.uuid, misp_name) # RESOLUTIONS - resolutions_iterator = self.client.iterator(f'/domains/{domain_report.id}/resolutions', limit=self.limit) + resolutions_iterator = self.client.iterator(f"/domains/{domain_report.id}/resolutions", limit=self.limit) for resolution in resolutions_iterator: - domain_object.add_attribute('ip', type='ip-dst', value=resolution.ip_address) + domain_object.add_attribute("ip", type="ip-dst", value=resolution.ip_address) # COMMUNICATING AND REFERRER FILES for relationship_name, misp_name in [ - ('communicating_files', 'communicates-with'), - ('referrer_files', 'referring') + ("communicating_files", "communicates-with"), + ("referrer_files", "referring"), ]: - files_iterator = self.client.iterator(f'/domains/{domain_report.id}/{relationship_name}', limit=self.limit) + files_iterator = self.client.iterator(f"/domains/{domain_report.id}/{relationship_name}", limit=self.limit) for file in files_iterator: file_object = self.create_misp_object(file) file_object.add_reference(domain_object.uuid, misp_name) @@ -139,16 +187,18 @@ def parse_domain(self, domain: str) -> str: return domain_object.uuid def parse_hash(self, file_hash: str) -> str: - file_report = self.client.get_object(f'/files/{file_hash}') + file_report = self.client.get_object(f"/files/{file_hash}") file_object = self.create_misp_object(file_report) # COMMUNICATING, DOWNLOADED AND REFERRER FILES for relationship_name, misp_name in [ - ('contacted_urls', 'communicates-with'), - ('contacted_domains', 'communicates-with'), - ('contacted_ips', 'communicates-with') + ("contacted_urls", "communicates-with"), + ("contacted_domains", "communicates-with"), + ("contacted_ips", "communicates-with"), ]: - related_files_iterator = self.client.iterator(f'/files/{file_report.id}/{relationship_name}', limit=self.limit) + related_files_iterator = self.client.iterator( + f"/files/{file_report.id}/{relationship_name}", limit=self.limit + ) for related_file in related_files_iterator: related_file_object = self.create_misp_object(related_file) related_file_object.add_reference(file_object.uuid, misp_name) @@ -158,29 +208,29 @@ def parse_hash(self, file_hash: str) -> str: return file_object.uuid def parse_ip(self, ip: str) -> str: - ip_report = self.client.get_object(f'/ip_addresses/{ip}') + ip_report = self.client.get_object(f"/ip_addresses/{ip}") # IP ip_object = self.create_misp_object(ip_report) # ASN - asn_object = MISPObject('asn') - asn_object.add_attribute('asn', type='AS', value=ip_report.asn) - asn_object.add_attribute('subnet-announced', type='ip-src', value=ip_report.network) - asn_object.add_attribute('country', type='text', value=ip_report.country) + asn_object = MISPObject("asn") + asn_object.add_attribute("asn", type="AS", value=ip_report.asn) + asn_object.add_attribute("subnet-announced", type="ip-src", value=ip_report.network) + asn_object.add_attribute("country", type="text", value=ip_report.country) self.misp_event.add_object(**asn_object) # RESOLUTIONS - resolutions_iterator = self.client.iterator(f'/ip_addresses/{ip_report.id}/resolutions', limit=self.limit) + resolutions_iterator = self.client.iterator(f"/ip_addresses/{ip_report.id}/resolutions", limit=self.limit) for resolution in resolutions_iterator: - ip_object.add_attribute('domain', type='domain', value=resolution.host_name) + ip_object.add_attribute("domain", type="domain", value=resolution.host_name) self.misp_event.add_object(**ip_object) return ip_object.uuid def parse_url(self, url: str) -> str: url_id = vt.url_id(url) - url_report = self.client.get_object(f'/urls/{url_id}') + url_report = self.client.get_object(f"/urls/{url_id}") url_object = self.create_misp_object(url_report) self.misp_event.add_object(**url_object) return url_object.uuid @@ -190,43 +240,42 @@ def get_proxy_settings(config: dict) -> dict: """Returns proxy settings in the requests format. If no proxy settings are set, return None.""" proxies = None - host = config.get('proxy_host') - port = config.get('proxy_port') - username = config.get('proxy_username') - password = config.get('proxy_password') + host = config.get("proxy_host") + port = config.get("proxy_port") + username = config.get("proxy_username") + password = config.get("proxy_password") if host: if not port: - misperrors['error'] = 'The virustotal_proxy_host config is set, ' \ - 'please also set the virustotal_proxy_port.' + misperrors["error"] = "The virustotal_proxy_host config is set, please also set the virustotal_proxy_port." raise KeyError parsed = urlparse(host) - if 'http' in parsed.scheme: - scheme = 'http' + if "http" in parsed.scheme: + scheme = "http" else: scheme = parsed.scheme netloc = parsed.netloc - host = f'{netloc}:{port}' + host = f"{netloc}:{port}" if username: if not password: - misperrors['error'] = 'The virustotal_proxy_username config is set, ' \ - 'please also set the virustotal_proxy_password.' + misperrors["error"] = ( + "The virustotal_proxy_username config is set, please also set the virustotal_proxy_password." + ) raise KeyError - auth = f'{username}:{password}' - host = auth + '@' + host + auth = f"{username}:{password}" + host = auth + "@" + host - proxies = { - 'http': f'{scheme}://{host}', - 'https': f'{scheme}://{host}' - } + proxies = {"http": f"{scheme}://{host}", "https": f"{scheme}://{host}"} return proxies def parse_error(status_code: int) -> str: - status_mapping = {204: 'VirusTotal request rate limit exceeded.', - 400: 'Incorrect request, please check the arguments.', - 403: 'You don\'t have enough privileges to make the request.'} + status_mapping = { + 204: "VirusTotal request rate limit exceeded.", + 400: "Incorrect request, please check the arguments.", + 403: "You don't have enough privileges to make the request.", + } if status_code in status_mapping: return status_mapping[status_code] return "VirusTotal may not be accessible." @@ -236,28 +285,30 @@ def handler(q=False): if q is False: return False request = json.loads(q) - if not request.get('config') or not request['config'].get('apikey'): - misperrors['error'] = 'A VirusTotal api key is required for this module.' + if not request.get("config") or not request["config"].get("apikey"): + misperrors["error"] = "A VirusTotal api key is required for this module." return misperrors - if not request.get('attribute') or not check_input_attribute(request['attribute']): - return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} - if request['attribute']['type'] not in mispattributes['input']: - return {'error': 'Unsupported attribute type.'} + if not request.get("attribute") or not check_input_attribute(request["attribute"]): + return {"error": f"{standard_error_message}, which should contain at least a type, a value and an uuid."} + if request["attribute"]["type"] not in mispattributes["input"]: + return {"error": "Unsupported attribute type."} - event_limit = request['config'].get('event_limit') - attribute = request['attribute'] - proxy_settings = get_proxy_settings(request.get('config')) + event_limit = request["config"].get("event_limit") + attribute = request["attribute"] + proxy_settings = get_proxy_settings(request.get("config")) try: - client = vt.Client(request['config']['apikey'], - headers={ - 'x-tool': 'MISPModuleVirusTotalPublicExpansion', - }, - proxy=proxy_settings['http'] if proxy_settings else None) + client = vt.Client( + request["config"]["apikey"], + headers={ + "x-tool": "MISPModuleVirusTotalPublicExpansion", + }, + proxy=proxy_settings["http"] if proxy_settings else None, + ) parser = VirusTotalParser(client, int(event_limit) if event_limit else None) parser.query_api(attribute) except vt.APIError as ex: - misperrors['error'] = ex.message + misperrors["error"] = ex.message return misperrors return parser.get_result() @@ -268,5 +319,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/virustotal_upload.py b/misp_modules/modules/expansion/virustotal_upload.py index e0ae14990..a627487ac 100644 --- a/misp_modules/modules/expansion/virustotal_upload.py +++ b/misp_modules/modules/expansion/virustotal_upload.py @@ -1,24 +1,24 @@ -import json -import sys import base64 +import hashlib import io +import json import zipfile + import requests -import hashlib -misperrors = {'error': 'Error'} -mispattributes = {'input': ['attachment', 'malware-sample'], 'output': ['link']} +misperrors = {"error": "Error"} +mispattributes = {"input": ["attachment", "malware-sample"], "output": ["link"]} moduleinfo = { - 'version': '1', - 'author': 'Karen Yousefi', - 'description': 'Module to push malware samples to VirusTotal', - 'module-type': ['expansion'], - 'name': 'VirusTotal Upload', - 'requirements': ['requests library'], - 'logo': 'virustotal.png', + "version": "1", + "author": "Karen Yousefi", + "description": "Module to push malware samples to VirusTotal", + "module-type": ["expansion"], + "name": "VirusTotal Upload", + "requirements": ["requests library"], + "logo": "virustotal.png", } -moduleconfig = ['virustotal_apikey'] +moduleconfig = ["virustotal_apikey"] def handler(q=False): @@ -28,7 +28,7 @@ def handler(q=False): try: data = request.get("data") - if 'malware-sample' in request: + if "malware-sample" in request: sample_filename = request.get("malware-sample").split("|", 1)[0] data = base64.b64decode(data) fl = io.BytesIO(data) @@ -36,14 +36,14 @@ def handler(q=False): sample_hashname = zf.namelist()[0] data = zf.read(sample_hashname, b"infected") zf.close() - elif 'attachment' in request: + elif "attachment" in request: sample_filename = request.get("attachment") data = base64.b64decode(data) else: - misperrors['error'] = "No malware sample or attachment supplied" + misperrors["error"] = "No malware sample or attachment supplied" return misperrors except Exception: - misperrors['error'] = "Unable to process submitted sample data" + misperrors["error"] = "Unable to process submitted sample data" return misperrors if request["config"].get("virustotal_apikey") is None: @@ -67,15 +67,15 @@ def handler(q=False): virustotal_link = f"https://www.virustotal.com/gui/file/{sha256}" except Exception as e: - misperrors['error'] = f"Unable to send sample to VirusTotal: {str(e)}" + misperrors["error"] = f"Unable to send sample to VirusTotal: {str(e)}" return misperrors r = { - 'results': [ + "results": [ { - 'types': 'link', - 'values': virustotal_link, - 'comment': 'Link to VirusTotal analysis', + "types": "link", + "values": virustotal_link, + "comment": "Link to VirusTotal analysis", } ] } @@ -87,5 +87,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/vmray_submit.py b/misp_modules/modules/expansion/vmray_submit.py index 78d7de531..c11f8a0a0 100644 --- a/misp_modules/modules/expansion/vmray_submit.py +++ b/misp_modules/modules/expansion/vmray_submit.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -''' +""" Submit sample to VMRay. Requires "vmray_rest_api" @@ -10,33 +10,47 @@ You can automate this by setting the PyMISP example script 'vmray_automation' as a cron job -''' +""" -import json import base64 -from distutils.util import strtobool - import io +import json import zipfile +from distutils.util import strtobool from _vmray.rest_api import VMRayRESTAPI -misperrors = {'error': 'Error'} -mispattributes = {'input': ['attachment', 'malware-sample'], 'output': ['text', 'sha1', 'sha256', 'md5', 'link']} +misperrors = {"error": "Error"} +mispattributes = { + "input": ["attachment", "malware-sample"], + "output": ["text", "sha1", "sha256", "md5", "link"], +} moduleinfo = { - 'version': '0.3', - 'author': 'Koen Van Impe', - 'description': 'Module to submit a sample to VMRay.', - 'module-type': ['expansion'], - 'name': 'VMRay Submit', - 'logo': 'vmray.png', - 'requirements': ['An access to the VMRay API (apikey & url)'], - 'features': 'This module takes an attachment or malware-sample attribute as input to query the VMRay API.\n\nThe sample contained within the attribute in then enriched with data from VMRay mapped into MISP attributes.', - 'references': ['https://www.vmray.com/'], - 'input': 'An attachment or malware-sample attribute.', - 'output': 'MISP attributes mapped from the result of the query on VMRay API, included in the following list:\n- text\n- sha1\n- sha256\n- md5\n- link', + "version": "0.3", + "author": "Koen Van Impe", + "description": "Module to submit a sample to VMRay.", + "module-type": ["expansion"], + "name": "VMRay Submit", + "logo": "vmray.png", + "requirements": ["An access to the VMRay API (apikey & url)"], + "features": ( + "This module takes an attachment or malware-sample attribute as input to query the VMRay API.\n\nThe sample" + " contained within the attribute in then enriched with data from VMRay mapped into MISP attributes." + ), + "references": ["https://www.vmray.com/"], + "input": "An attachment or malware-sample attribute.", + "output": ( + "MISP attributes mapped from the result of the query on VMRay API, included in the following list:\n- text\n-" + " sha1\n- sha256\n- md5\n- link" + ), } -moduleconfig = ['apikey', 'url', 'shareable', 'do_not_reanalyze', 'do_not_include_vmrayjobids'] +moduleconfig = [ + "apikey", + "url", + "shareable", + "do_not_reanalyze", + "do_not_include_vmrayjobids", +] include_vmrayjobids = False @@ -51,7 +65,7 @@ def handler(q=False): try: data = request.get("data") - if 'malware-sample' in request: + if "malware-sample" in request: # malicious samples are encrypted with zip (password infected) and then base64 encoded sample_filename = request.get("malware-sample").split("|", 1)[0] data = base64.b64decode(data) @@ -60,16 +74,16 @@ def handler(q=False): sample_hashname = zf.namelist()[0] data = zf.read(sample_hashname, b"infected") zf.close() - elif 'attachment' in request: + elif "attachment" in request: # All attachments get base64 encoded sample_filename = request.get("attachment") data = base64.b64decode(data) else: - misperrors['error'] = "No malware sample or attachment supplied" + misperrors["error"] = "No malware sample or attachment supplied" return misperrors except Exception: - misperrors['error'] = "Unable to process submited sample data" + misperrors["error"] = "Unable to process submited sample data" return misperrors if (request["config"].get("apikey") is None) or (request["config"].get("url") is None): @@ -83,8 +97,8 @@ def handler(q=False): do_not_include_vmrayjobids = request["config"].get("do_not_include_vmrayjobids") try: - shareable = bool(strtobool(shareable)) # Do we want the sample to be shared? - reanalyze = not bool(strtobool(do_not_reanalyze)) # Always reanalyze the sample? + shareable = bool(strtobool(shareable)) # Do we want the sample to be shared? + reanalyze = not bool(strtobool(do_not_reanalyze)) # Always reanalyze the sample? include_vmrayjobids = not bool(strtobool(do_not_include_vmrayjobids)) # Include the references to VMRay job IDs except ValueError: misperrors["error"] = "Error while processing settings. Please double-check your values." @@ -93,21 +107,21 @@ def handler(q=False): if data and sample_filename: args = {} args["shareable"] = shareable - args["sample_file"] = {'data': io.BytesIO(data), 'filename': sample_filename} + args["sample_file"] = {"data": io.BytesIO(data), "filename": sample_filename} args["reanalyze"] = reanalyze try: vmraydata = vmraySubmit(api, args) if vmraydata["errors"] and "Submission not stored" not in vmraydata["errors"][0]["error_msg"]: - misperrors['error'] = "VMRay: %s" % vmraydata["errors"][0]["error_msg"] + misperrors["error"] = "VMRay: %s" % vmraydata["errors"][0]["error_msg"] return misperrors else: return vmrayProcess(vmraydata) except Exception: - misperrors['error'] = "Problem when calling API." + misperrors["error"] = "Problem when calling API." return misperrors else: - misperrors['error'] = "No sample data or filename." + misperrors["error"] = "No sample data or filename." return misperrors @@ -116,12 +130,12 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo def vmrayProcess(vmraydata): - ''' Process the JSON file returned by vmray''' + """Process the JSON file returned by vmray""" if vmraydata: try: sample = vmraydata["samples"][0] @@ -129,12 +143,18 @@ def vmrayProcess(vmraydata): # Result received? if sample: - r = {'results': []} - r['results'].append({'types': 'md5', 'values': sample['sample_md5hash']}) - r['results'].append({'types': 'sha1', 'values': sample['sample_sha1hash']}) - r['results'].append({'types': 'sha256', 'values': sample['sample_sha256hash']}) - r['results'].append({'types': 'text', 'values': 'VMRay Sample ID: %s' % sample['sample_id'], 'tags': 'workflow:state="incomplete"'}) - r['results'].append({'types': 'link', 'values': sample['sample_webif_url']}) + r = {"results": []} + r["results"].append({"types": "md5", "values": sample["sample_md5hash"]}) + r["results"].append({"types": "sha1", "values": sample["sample_sha1hash"]}) + r["results"].append({"types": "sha256", "values": sample["sample_sha256hash"]}) + r["results"].append( + { + "types": "text", + "values": "VMRay Sample ID: %s" % sample["sample_id"], + "tags": 'workflow:state="incomplete"', + } + ) + r["results"].append({"types": "link", "values": sample["sample_webif_url"]}) # Include data from different jobs if include_vmrayjobids and len(jobs) > 0: @@ -142,20 +162,25 @@ def vmrayProcess(vmraydata): job_id = job["job_id"] job_vm_name = job["job_vm_name"] job_configuration_name = job["job_configuration_name"] - r["results"].append({"types": "text", "values": "VMRay Job ID %s (%s - %s)" % (job_id, job_vm_name, job_configuration_name)}) + r["results"].append( + { + "types": "text", + "values": "VMRay Job ID %s (%s - %s)" % (job_id, job_vm_name, job_configuration_name), + } + ) return r else: - misperrors['error'] = "No valid results returned." + misperrors["error"] = "No valid results returned." return misperrors except Exception: - misperrors['error'] = "No valid submission data returned." + misperrors["error"] = "No valid submission data returned." return misperrors else: - misperrors['error'] = "Unable to parse results." + misperrors["error"] = "Unable to parse results." return misperrors def vmraySubmit(api, args): - ''' Submit the sample to VMRay''' + """Submit the sample to VMRay""" vmraydata = api.call("POST", "/rest/sample/submit", args) return vmraydata diff --git a/misp_modules/modules/expansion/vmware_nsx.py b/misp_modules/modules/expansion/vmware_nsx.py index 45adcbbed..b9ccbf886 100644 --- a/misp_modules/modules/expansion/vmware_nsx.py +++ b/misp_modules/modules/expansion/vmware_nsx.py @@ -11,17 +11,15 @@ import ipaddress import json import logging -import pymisp import sys -import vt import zipfile -from urllib import parse from typing import Any, Dict, List, Optional, Tuple, Union +from urllib import parse +import pymisp import tau_clients -from tau_clients import exceptions -from tau_clients import nsx_defender - +import vt +from tau_clients import exceptions, nsx_defender logger = logging.getLogger("vmware_nsx") logger.setLevel(logging.DEBUG) @@ -43,28 +41,31 @@ } moduleinfo = { - 'version': '0.2', - 'author': 'Jason Zhang, Stefano Ortolani', - 'description': 'Module to enrich a file or URL with VMware NSX Defender.', - 'module-type': ['expansion', 'hover'], - 'name': 'VMware NSX Defender Enrich', - 'logo': 'vmware_nsx.png', - 'requirements': ['The module requires a VMware NSX Defender Analysis `api_token` and `key`.'], - 'features': 'This module takes an IoC such as file hash, file attachment, malware-sample or url as input to query VMware NSX Defender.\n\nThe IoC is then enriched with data from VMware NSX Defender.', - 'references': ['https://www.vmware.com'], - 'input': 'File hash, attachment or URL to be enriched with VMware NSX Defender.', - 'output': 'Objects and tags generated by VMware NSX Defender.', + "version": "0.2", + "author": "Jason Zhang, Stefano Ortolani", + "description": "Module to enrich a file or URL with VMware NSX Defender.", + "module-type": ["expansion", "hover"], + "name": "VMware NSX Defender Enrich", + "logo": "vmware_nsx.png", + "requirements": ["The module requires a VMware NSX Defender Analysis `api_token` and `key`."], + "features": ( + "This module takes an IoC such as file hash, file attachment, malware-sample or url as input to query VMware" + " NSX Defender.\n\nThe IoC is then enriched with data from VMware NSX Defender." + ), + "references": ["https://www.vmware.com"], + "input": "File hash, attachment or URL to be enriched with VMware NSX Defender.", + "output": "Objects and tags generated by VMware NSX Defender.", } moduleconfig = [ - "analysis_url", # optional, defaults to hard-coded values - "analysis_verify_ssl", # optional, defaults to True - "analysis_key", # required - "analysis_api_token", # required - "vt_key", # optional - "misp_url", # optional - "misp_verify_ssl", # optional, defaults to True - "misp_key", # optional + "analysis_url", # optional, defaults to hard-coded values + "analysis_verify_ssl", # optional, defaults to True + "analysis_key", # required + "analysis_api_token", # required + "vt_key", # optional + "misp_url", # optional + "misp_verify_ssl", # optional, defaults to True + "misp_key", # optional ] DEFAULT_ZIP_PASSWORD = b"infected" @@ -111,7 +112,7 @@ def parse(self, analysis_link: str, result: Dict[str, Any]) -> pymisp.MISPEvent: "mimetype", category="Payload delivery", type="mime-type", - value=result["analysis_subject"]["mime_type"] + value=result["analysis_subject"]["mime_type"], ) misp_event.add_object(o) @@ -175,7 +176,7 @@ def parse(self, analysis_link: str, result: Dict[str, Any]) -> pymisp.MISPEvent: uri = "http://{}:{}{}".format( http_conversation["dst_host"], http_conversation["dst_port"], - path + path, ) o = pymisp.MISPObject(name="http-request") o.add_attribute("host", http_conversation["dst_host"]) @@ -207,7 +208,10 @@ def parse(self, analysis_link: str, result: Dict[str, Any]) -> pymisp.MISPEvent: # Add mitre techniques for techniques in result.get("activity_to_mitre_techniques", {}).values(): for technique in techniques: - for misp_technique_id, misp_technique_name in self.techniques_galaxy.items(): + for ( + misp_technique_id, + misp_technique_name, + ) in self.techniques_galaxy.items(): if technique["id"].casefold() in misp_technique_id.casefold(): # If report details a sub-technique, trust the match # Otherwise trust it only if the MISP technique is not a sub-technique @@ -325,6 +329,7 @@ def _get_latest_analysis( :raises exceptions.ApiError: in case of client errors :raises exceptions.CommunicationError: in case of client communication errors """ + def _parse_expiration(task_info: Dict[str, str]) -> datetime.datetime: """ Parse expiration time of a task @@ -334,6 +339,7 @@ def _parse_expiration(task_info: Dict[str, str]) -> datetime.datetime: :return: the parsed datetime object """ return datetime.datetime.strptime(task_info["expires"], "%Y-%m-%d %H:%M:%S") + results = [] for data_center, client in clients.items(): response = client.query_file_hash(file_hash=file_hash) @@ -421,7 +427,8 @@ def handler(q: Union[bool, str] = False) -> Union[bool, Dict[str, Any]]: api_url=tau_clients.NSX_DEFENDER_ANALYSIS_URLS[data_center], login_params=login_params, verify_ssl=bool(config.get("analysis_verify_ssl", True)), - ) for data_center in [ + ) + for data_center in [ tau_clients.NSX_DEFENDER_DC_WESTUS, tau_clients.NSX_DEFENDER_DC_NLEMEA, ] @@ -577,7 +584,7 @@ def main(): "attribute": { "type": "url", "value": "https://www.google.com", - } + }, } ) print(json.dumps(handler(j), indent=4, sort_keys=True)) @@ -593,7 +600,7 @@ def main(): "type": "attachment", "value": "test.docx", "data": base64.b64encode(data).decode("utf-8"), - } + }, } ) print(json.dumps(handler(j), indent=4, sort_keys=True)) @@ -605,7 +612,7 @@ def main(): "attribute": { "type": "md5", "value": "002c56165a0e78369d0e1023ce044bf0", - } + }, } ) print(json.dumps(handler(j), indent=4, sort_keys=True)) @@ -617,7 +624,7 @@ def main(): "attribute": { "type": "sha1", "value": "2aac25ecdccf87abf6f1651ef2ffb30fcf732250", - } + }, } ) print(json.dumps(handler(j), indent=4, sort_keys=True)) diff --git a/misp_modules/modules/expansion/vulndb.py b/misp_modules/modules/expansion/vulndb.py index f467cafc0..dd14b6c80 100644 --- a/misp_modules/modules/expansion/vulndb.py +++ b/misp_modules/modules/expansion/vulndb.py @@ -1,46 +1,56 @@ #!/usr/bin/env python3 -''' +""" Import VulnDB https://vulndb.cyberriskanalytics.com/ https://www.riskbasedsecurity.com/ -''' +""" -import oauth2 as oauth import json - import logging import sys +import oauth2 as oauth -log = logging.getLogger('vulndb') +log = logging.getLogger("vulndb") log.setLevel(logging.DEBUG) ch = logging.StreamHandler(sys.stdout) ch.setLevel(logging.DEBUG) -formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") ch.setFormatter(formatter) log.addHandler(ch) -misperrors = {'error': 'Error'} -mispattributes = { - 'input': ['vulnerability'], - 'output': ['text', 'link', 'cpe']} +misperrors = {"error": "Error"} +mispattributes = {"input": ["vulnerability"], "output": ["text", "link", "cpe"]} moduleinfo = { - 'version': '0.1', - 'author': 'Koen Van Impe', - 'description': 'Module to query VulnDB (RiskBasedSecurity.com).', - 'module-type': ['expansion', 'hover'], - 'name': 'VulnDB Lookup', - 'logo': 'vulndb.png', - 'requirements': ['An access to the VulnDB API (apikey, apisecret)'], - 'features': 'This module takes a vulnerability attribute as input and queries VulnDB in order to get some additional data about it.\n\nThe API gives the result of the query which can be displayed in the screen, and/or mapped into MISP attributes to add in the event.', - 'references': ['https://vulndb.cyberriskanalytics.com/'], - 'input': 'A vulnerability attribute.', - 'output': 'Additional data enriching the CVE input, fetched from VulnDB.', + "version": "0.1", + "author": "Koen Van Impe", + "description": "Module to query VulnDB (RiskBasedSecurity.com).", + "module-type": ["expansion", "hover"], + "name": "VulnDB Lookup", + "logo": "vulndb.png", + "requirements": ["An access to the VulnDB API (apikey, apisecret)"], + "features": ( + "This module takes a vulnerability attribute as input and queries VulnDB in order to get some additional data" + " about it.\n\nThe API gives the result of the query which can be displayed in the screen, and/or mapped into" + " MISP attributes to add in the event." + ), + "references": ["https://vulndb.cyberriskanalytics.com/"], + "input": "A vulnerability attribute.", + "output": "Additional data enriching the CVE input, fetched from VulnDB.", } -moduleconfig = ['apikey', 'apisecret', 'discard_dates', 'discard_external_references', 'discard_cvss', 'discard_productinformation', 'discard_classification', 'discard_cpe'] +moduleconfig = [ + "apikey", + "apisecret", + "discard_dates", + "discard_external_references", + "discard_cvss", + "discard_productinformation", + "discard_classification", + "discard_cpe", +] def handler(q=False): @@ -52,10 +62,10 @@ def handler(q=False): request = json.loads(q) # Only continue if we have a vulnerability attribute - if not request.get('vulnerability'): - misperrors['error'] = 'Vulnerability ID missing for VulnDB.' + if not request.get("vulnerability"): + misperrors["error"] = "Vulnerability ID missing for VulnDB." return misperrors - vulnerability = request.get('vulnerability') + vulnerability = request.get("vulnerability") if request["config"].get("apikey") is None or request["config"].get("apisecret") is None: misperrors["error"] = "Missing API key or secret value for VulnDB." @@ -73,13 +83,22 @@ def handler(q=False): if request["config"].get("discard_dates") is not None and request["config"].get("discard_dates").lower() == "true": add_dates = False - if request["config"].get("discard_external_references") is not None and request["config"].get("discard_external_references").lower() == "true": + if ( + request["config"].get("discard_external_references") is not None + and request["config"].get("discard_external_references").lower() == "true" + ): add_ext_references = False if request["config"].get("discard_cvss") is not None and request["config"].get("discard_cvss").lower() == "true": add_cvss = False - if request["config"].get("discard_productinformation") is not None and request["config"].get("discard_productinformation").lower() == "true": + if ( + request["config"].get("discard_productinformation") is not None + and request["config"].get("discard_productinformation").lower() == "true" + ): add_products = False - if request["config"].get("discard_classification") is not None and request["config"].get("discard_classification").lower() == "true": + if ( + request["config"].get("discard_classification") is not None + and request["config"].get("discard_classification").lower() == "true" + ): add_classifications = False if request["config"].get("discard_cpe") is not None and request["config"].get("discard_cpe").lower() == "true": add_cpe = False @@ -88,7 +107,11 @@ def handler(q=False): if add_cpe: cpu_vulndb = "?show_cpe=true" - find_by_cve_url = "%s/api/v1/vulnerabilities/%s/find_by_cve_id%s" % (VULNDB_URL, vulnerability, cpu_vulndb) + find_by_cve_url = "%s/api/v1/vulnerabilities/%s/find_by_cve_id%s" % ( + VULNDB_URL, + vulnerability, + cpu_vulndb, + ) log.debug(find_by_cve_url) try: @@ -99,11 +122,11 @@ def handler(q=False): content_json = json.loads(content.decode()) if content_json: - if 'error' in content_json: + if "error" in content_json: misperrors["error"] = "No CVE information found." return misperrors else: - output = {'results': list()} + output = {"results": list()} values_text = list() values_links = list() values_cpe = list() @@ -112,15 +135,18 @@ def handler(q=False): # Include the VulnDB title and ID values_text.append(results["title"]) - vulndb_id_link = "%s/vulnerabilities/%s" % (VULNDB_URL, results["vulndb_id"]) + vulndb_id_link = "%s/vulnerabilities/%s" % ( + VULNDB_URL, + results["vulndb_id"], + ) values_links.append(vulndb_id_link) # Descriptive part of the VulnDB item - description = results.get('description', '') or '' - keywords = results.get('keywords', '') or '' - solution = results.get('solution', '') or '' - manual_notes = results.get('manual_notes', '') or '' - t_description = results.get('t_description', '') or '' + description = results.get("description", "") or "" + keywords = results.get("keywords", "") or "" + solution = results.get("solution", "") or "" + manual_notes = results.get("manual_notes", "") or "" + t_description = results.get("t_description", "") or "" if description: values_text.append(description) if t_description: @@ -135,31 +161,31 @@ def handler(q=False): # VulnDB items contain a number of dates, do we include them? if add_dates: log.debug("Include dates") - solution_date = results.get('solution_date', '') or '' + solution_date = results.get("solution_date", "") or "" if solution_date: values_text.append("Solution date: " + solution_date) - disclosure_date = results.get('disclosure_date', '') or '' + disclosure_date = results.get("disclosure_date", "") or "" if disclosure_date: values_text.append("Disclosure date: " + disclosure_date) - discovery_date = results.get('discovery_date', '') or '' + discovery_date = results.get("discovery_date", "") or "" if discovery_date: values_text.append("Discovery date: " + discovery_date) - exploit_publish_date = results.get('exploit_publish_date', '') or '' + exploit_publish_date = results.get("exploit_publish_date", "") or "" if exploit_publish_date: values_text.append("Exploit published date: " + exploit_publish_date) - vendor_informed_date = results.get('vendor_informed_date', '') or '' + vendor_informed_date = results.get("vendor_informed_date", "") or "" if vendor_informed_date: values_text.append("Vendor informed date: " + vendor_informed_date) - vendor_ack_date = results.get('vendor_ack_date', '') or '' + vendor_ack_date = results.get("vendor_ack_date", "") or "" if vendor_ack_date: values_text.append("Vendor acknowledgement date: " + vendor_ack_date) - third_party_solution_date = results.get('third_party_solution_date', '') or '' + third_party_solution_date = results.get("third_party_solution_date", "") or "" if third_party_solution_date: values_text.append("Third party solution date: " + third_party_solution_date) # External references if add_ext_references: - ext_references = results.get('ext_references') + ext_references = results.get("ext_references") if ext_references: log.debug("Include external references") for reference in ext_references: @@ -207,21 +233,28 @@ def handler(q=False): # CVSS Scoring if add_cvss: - cvss = results.get('cvss_metrics') + cvss = results.get("cvss_metrics") if cvss: log.debug("Include CVSS") for cvss_metric in cvss: score = cvss_metric.get("score") if score: - values_text.append("CVSS %s (base: %s) (source: %s)" % (score, cvss_metric.get("calculated_cvss_base_score"), cvss_metric.get("source"))) + values_text.append( + "CVSS %s (base: %s) (source: %s)" + % ( + score, + cvss_metric.get("calculated_cvss_base_score"), + cvss_metric.get("source"), + ) + ) # Add products if add_products: - products = results.get('products') + products = results.get("products") if products and len(products) > 0: # Get the vendors - vendors = results.get('vendors') + vendors = results.get("vendors") vendors_name = "" log.debug("Include product information") if vendors: @@ -266,14 +299,14 @@ def handler(q=False): for classification in classifications: longname = classification.get("longname") description = classification.get("description") - vulnerability_classification += " \"%s\" " % longname + vulnerability_classification += ' "%s" ' % longname values_text.append(vulnerability_classification) # Finished processing the VulnDB reply; set the result for MISP - output['results'] += [{'types': 'text', 'values': values_text}] - output['results'] += [{'types': 'link', 'values': values_links}] + output["results"] += [{"types": "text", "values": values_text}] + output["results"] += [{"types": "link", "values": values_links}] if add_cpe: - output['results'] += [{'types': 'cpe', 'values': values_cpe}] + output["results"] += [{"types": "cpe", "values": values_cpe}] return output else: misperrors["error"] = "No information retrieved from VulnDB." @@ -288,5 +321,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/vulnerability_lookup.py b/misp_modules/modules/expansion/vulnerability_lookup.py index f457883d9..c3b9e309d 100644 --- a/misp_modules/modules/expansion/vulnerability_lookup.py +++ b/misp_modules/modules/expansion/vulnerability_lookup.py @@ -1,47 +1,50 @@ import json + import requests + from . import check_input_attribute, standard_error_message from ._vulnerability_parser.vulnerability_parser import VulnerabilityLookupParser -misperrors = {'error': 'Error'} -mispattributes = {'input': ['vulnerability'], 'format': 'misp_standard'} +misperrors = {"error": "Error"} +mispattributes = {"input": ["vulnerability"], "format": "misp_standard"} moduleinfo = { - 'version': '2', - 'author': 'Christian Studer', - 'description': 'An expansion module to query Vulnerability Lookup', - 'module-type': ['expansion', 'hover'], - 'name': 'Vulnerability Lookup', - 'logo': 'vulnerability_lookup.png', - 'requirements': [], - 'features': 'The module takes a vulnerability attribute as input and queries Vulnerability Lookup to gather additional information based on the Vulnerability ID. The result of the query is then parsed and converted into MISP content which can be added to the original event to enrich the input attribute.', - 'references': ['https://vulnerability.circl.lu'], - 'input': 'Vulnerability Attribute', - 'output': 'Additional information on the vulnerability, gathered from the Vulnerability Lookup API.', + "version": "2", + "author": "Christian Studer", + "description": "An expansion module to query Vulnerability Lookup", + "module-type": ["expansion", "hover"], + "name": "Vulnerability Lookup", + "logo": "vulnerability_lookup.png", + "requirements": [], + "features": ( + "The module takes a vulnerability attribute as input and queries Vulnerability Lookup to gather additional" + " information based on the Vulnerability ID. The result of the query is then parsed and converted into MISP" + " content which can be added to the original event to enrich the input attribute." + ), + "references": ["https://vulnerability.circl.lu"], + "input": "Vulnerability Attribute", + "output": "Additional information on the vulnerability, gathered from the Vulnerability Lookup API.", } -api_url = 'https://vulnerability.circl.lu' +api_url = "https://vulnerability.circl.lu" def handler(q=False): if q is False: return q request = json.loads(q) - if not check_input_attribute(request.get('attribute', {})): - return { - 'error': f'{standard_error_message}, which should contain ' - 'at least a type, a value and an UUID.' - } - attribute = request['attribute'] - if attribute.get('type') != 'vulnerability': - misperrors['error'] = 'Vulnerability ID missing' + if not check_input_attribute(request.get("attribute", {})): + return {"error": f"{standard_error_message}, which should contain at least a type, a value and an UUID."} + attribute = request["attribute"] + if attribute.get("type") != "vulnerability": + misperrors["error"] = "Vulnerability ID missing" return misperrors lookup = requests.get(f"{api_url}/api/vulnerability/{attribute['value']}") if lookup.status_code == 200: vulnerability = lookup.json() if not vulnerability: - misperrors['error'] = 'Non existing Vulnerability ID.' + misperrors["error"] = "Non existing Vulnerability ID." return misperrors else: - misperrors['error'] = 'Vulnerability Lookup API not accessible.' + misperrors["error"] = "Vulnerability Lookup API not accessible." return misperrors parser = VulnerabilityLookupParser(attribute) parser.parse_lookup_result(vulnerability, api_url) diff --git a/misp_modules/modules/expansion/vulners.py b/misp_modules/modules/expansion/vulners.py index 5c10575de..3f4853636 100644 --- a/misp_modules/modules/expansion/vulners.py +++ b/misp_modules/modules/expansion/vulners.py @@ -1,20 +1,24 @@ import json + import vulners -misperrors = {'error': 'Error'} -mispattributes = {'input': ['vulnerability'], 'output': ['text']} +misperrors = {"error": "Error"} +mispattributes = {"input": ["vulnerability"], "output": ["text"]} moduleinfo = { - 'version': '0.1', - 'author': 'Igor Ivanov', - 'description': 'An expansion hover module to expand information about CVE id using Vulners API.', - 'module-type': ['hover'], - 'name': 'Vulners Lookup', - 'logo': 'vulners.png', - 'requirements': ['Vulners python library', 'An access to the Vulners API'], - 'features': 'This module takes a vulnerability attribute as input and queries the Vulners API in order to get some additional data about it.\n\nThe API then returns details about the vulnerability.', - 'references': ['https://vulners.com/'], - 'input': 'A vulnerability attribute.', - 'output': 'Text giving additional information about the CVE in input.', + "version": "0.1", + "author": "Igor Ivanov", + "description": "An expansion hover module to expand information about CVE id using Vulners API.", + "module-type": ["hover"], + "name": "Vulners Lookup", + "logo": "vulners.png", + "requirements": ["Vulners python library", "An access to the Vulners API"], + "features": ( + "This module takes a vulnerability attribute as input and queries the Vulners API in order to get some" + " additional data about it.\n\nThe API then returns details about the vulnerability." + ), + "references": ["https://vulners.com/"], + "input": "A vulnerability attribute.", + "output": "Text giving additional information about the CVE in input.", } # Get API key from https://vulners.com/userinfo @@ -25,26 +29,26 @@ def handler(q=False): if q is False: return False request = json.loads(q) - if not request.get('vulnerability'): - misperrors['error'] = 'Vulnerability id missing' + if not request.get("vulnerability"): + misperrors["error"] = "Vulnerability id missing" return misperrors - ai_summary = '' - exploit_summary = '' - vuln_summary = '' + ai_summary = "" + exploit_summary = "" + vuln_summary = "" - if not request.get('config') or not request['config'].get('apikey'): - return {'error': "A Vulners api key is required for this module."} + if not request.get("config") or not request["config"].get("apikey"): + return {"error": "A Vulners api key is required for this module."} - key = request['config']['apikey'] + key = request["config"]["apikey"] vulners_api = vulners.Vulners(api_key=key) - vulnerability = request.get('vulnerability') + vulnerability = request.get("vulnerability") vulners_document = vulners_api.document(vulnerability) # Get AI scoring from the document if it's already calculated # There is no need to call AI Scoring method - if 'score' in vulners_document.get('enchantments', {}): - vulners_ai_score = vulners_document['enchantments']['score']['value'] + if "score" in vulners_document.get("enchantments", {}): + vulners_ai_score = vulners_document["enchantments"]["score"]["value"] else: vulners_ai_score = vulners_api.get_ai_score(vulnerability) if len(vulners_ai_score) == 2: @@ -53,22 +57,22 @@ def handler(q=False): vulners_exploits = vulners_api.searchExploit(vulnerability) if vulners_document: - vuln_summary += vulners_document.get('description') + vuln_summary += vulners_document.get("description") else: - vuln_summary += 'Non existing CVE' + vuln_summary += "Non existing CVE" if vulners_ai_score: - ai_summary += 'Vulners AI Score is ' + str(vulners_ai_score) + " " + ai_summary += "Vulners AI Score is " + str(vulners_ai_score) + " " if vulners_exploits: exploit_summary += " || " + str(len(vulners_exploits)) + " Public exploits available:\n " for exploit in vulners_exploits: - exploit_summary += exploit['title'] + " " + exploit['href'] + "\n " + exploit_summary += exploit["title"] + " " + exploit["href"] + "\n " exploit_summary += "|| Vulnerability Description: " + vuln_summary summary = ai_summary + exploit_summary + vuln_summary - r = {'results': [{'types': mispattributes['output'], 'values': summary}]} + r = {"results": [{"types": mispattributes["output"], "values": summary}]} return r @@ -77,5 +81,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/vysion.py b/misp_modules/modules/expansion/vysion.py index 35fc56cc2..8acf5320f 100644 --- a/misp_modules/modules/expansion/vysion.py +++ b/misp_modules/modules/expansion/vysion.py @@ -1,13 +1,12 @@ import json -from pymisp import MISPAttribute, MISPEvent -from urllib.parse import urlparse - import logging +from urllib.parse import urlparse import vysion.client as vysion - -import vysion.dto as dto +import vysion.dto +from pymisp import MISPAttribute, MISPEvent from vysion.dto.util import MISPProcessor + from . import standard_error_message misperrors = {"error": "Error"} @@ -29,17 +28,25 @@ # possible module-types: 'expansion', 'hover' or both moduleinfo = { - 'version': '1', - 'author': 'Byron Labs', - 'description': 'Module to enrich the information by making use of the Vysion API.', - 'module-type': ['expansion'], - 'name': 'Vysion Enrich', - 'logo': 'vysion.png', - 'requirements': ['Vysion python library', 'Vysion API Key'], - 'features': "This module gets correlated information from Byron Labs' dark web intelligence database. With this you will get several objects containing information related to, for example, an organization victim of a ransomware attack.", - 'references': ['https://vysion.ai/', 'https://developers.vysion.ai/', 'https://github.com/ByronLabs/vysion-cti/tree/main'], - 'input': 'company(target-org), country, info, BTC, XMR and DASH address.', - 'output': 'MISP objects containing title, link to our webapp and TOR, i2p or clearnet URLs.', + "version": "1", + "author": "Byron Labs", + "description": "Module to enrich the information by making use of the Vysion API.", + "module-type": ["expansion"], + "name": "Vysion Enrich", + "logo": "vysion.png", + "requirements": ["Vysion python library", "Vysion API Key"], + "features": ( + "This module gets correlated information from Byron Labs' dark web intelligence database. With this you will" + " get several objects containing information related to, for example, an organization victim of a ransomware" + " attack." + ), + "references": [ + "https://vysion.ai/", + "https://developers.vysion.ai/", + "https://github.com/ByronLabs/vysion-cti/tree/main", + ], + "input": "company(target-org), country, info, BTC, XMR and DASH address.", + "output": "MISP objects containing title, link to our webapp and TOR, i2p or clearnet URLs.", } # config fields that your code expects from the site admin @@ -70,10 +77,7 @@ def get_proxy_settings(config: dict) -> dict: if host: if not port: - misperrors["error"] = ( - "The vysion_proxy_host config is set, " - "please also set the vysion_proxy_port." - ) + misperrors["error"] = "The vysion_proxy_host config is set, please also set the vysion_proxy_port." raise KeyError parsed = urlparse(host) if "http" in parsed.scheme: @@ -86,8 +90,7 @@ def get_proxy_settings(config: dict) -> dict: if username: if not password: misperrors["error"] = ( - "The vysion_proxy_username config is set, " - "please also set the vysion_proxy_password." + "The vysion_proxy_username config is set, please also set the vysion_proxy_password." ) raise KeyError auth = f"{username}:{password}" @@ -123,9 +126,7 @@ def handler(q=False): return misperrors if not request.get("attribute"): - return { - "error": f"{standard_error_message}, which should contain at least a type, a value and an uuid." - } + return {"error": f"{standard_error_message}, which should contain at least a type, a value and an uuid."} if request["attribute"]["type"] not in mispattributes["input"]: return {"error": "Unsupported attribute type."} @@ -171,11 +172,11 @@ def handler(q=False): elif attribute_type == "phone-number": result = client.search(attribute_value) elif attribute_type == "btc": - result = client.find_wallet("BTC",attribute_value) + result = client.find_wallet("BTC", attribute_value) elif attribute_type == "xmr": - result = client.find_wallet("XMR",attribute_value) + result = client.find_wallet("XMR", attribute_value) elif attribute_type == "dash": - result = client.find_wallet("DASH",attribute_value) + result = client.find_wallet("DASH", attribute_value) if result is None: return {"results": {}} @@ -189,17 +190,9 @@ def handler(q=False): return { "results": { - "Object": [ - json.loads(object.to_json()) for object in misp_event.objects - ], - "Attribute": [ - json.loads(attribute.to_json()) - for attribute in misp_event.attributes - ], - "Tag": [ - json.loads(tag.to_json()) - for tag in misp_event.tags - ] + "Object": [json.loads(object.to_json()) for object in misp_event.objects], + "Attribute": [json.loads(attribute.to_json()) for attribute in misp_event.attributes], + "Tag": [json.loads(tag.to_json()) for tag in misp_event.tags], } } @@ -218,4 +211,4 @@ def introspection(): def version(): moduleinfo["config"] = moduleconfig - return moduleinfo \ No newline at end of file + return moduleinfo diff --git a/misp_modules/modules/expansion/whois.py b/misp_modules/modules/expansion/whois.py index acec4aef7..3d454ffbe 100644 --- a/misp_modules/modules/expansion/whois.py +++ b/misp_modules/modules/expansion/whois.py @@ -3,64 +3,62 @@ import json import socket -misperrors = {'error': 'Error'} -mispattributes = {'input': ['domain', 'ip-src', 'ip-dst'], 'output': ['freetext']} +misperrors = {"error": "Error"} +mispattributes = {"input": ["domain", "ip-src", "ip-dst"], "output": ["freetext"]} moduleinfo = { - 'version': '0.1', - 'author': 'Raphaël Vinot', - 'description': 'Module to query a local instance of uwhois (https://github.com/rafiot/uwhoisd).', - 'module-type': ['expansion'], - 'name': 'Whois Lookup', - 'logo': '', - 'requirements': ['uwhois: A whois python library'], - 'features': "This module takes a domain or IP address attribute as input and queries a 'Univseral Whois proxy server' to get the correct details of the Whois query on the input value (check the references for more details about this whois server).", - 'references': ['https://github.com/Lookyloo/uwhoisd'], - 'input': 'A domain or IP address attribute.', - 'output': 'Text describing the result of a whois request for the input value.', + "version": "0.1", + "author": "Raphaël Vinot", + "description": "Module to query a local instance of uwhois (https://github.com/rafiot/uwhoisd).", + "module-type": ["expansion"], + "name": "Whois Lookup", + "logo": "", + "requirements": ["uwhois: A whois python library"], + "features": ( + "This module takes a domain or IP address attribute as input and queries a 'Univseral Whois proxy server' to" + " get the correct details of the Whois query on the input value (check the references for more details about" + " this whois server)." + ), + "references": ["https://github.com/Lookyloo/uwhoisd"], + "input": "A domain or IP address attribute.", + "output": "Text describing the result of a whois request for the input value.", } -moduleconfig = ['server', 'port'] +moduleconfig = ["server", "port"] def handler(q=False): if q is False: return False request = json.loads(q) - if request.get('domain'): - toquery = request['domain'] - elif request.get('ip-src'): - toquery = request['ip-src'] - elif request.get('ip-dst'): - toquery = request['ip-dst'] + if request.get("domain"): + toquery = request["domain"] + elif request.get("ip-src"): + toquery = request["ip-src"] + elif request.get("ip-dst"): + toquery = request["ip-dst"] else: - misperrors['error'] = "Unsupported attributes type" + misperrors["error"] = "Unsupported attributes type" return misperrors - if not request.get('config') or ( - not request['config'].get('server') and not request['config'].get('port') - ): - misperrors['error'] = 'Whois local instance address is missing' + if not request.get("config") or (not request["config"].get("server") and not request["config"].get("port")): + misperrors["error"] = "Whois local instance address is missing" return misperrors - if 'event_id' in request: - return handle_expansion( - request['config']['server'], int(request['config']['port']), toquery - ) + if "event_id" in request: + return handle_expansion(request["config"]["server"], int(request["config"]["port"]), toquery) def handle_expansion(server, port, query): - bytes_whois = b'' + bytes_whois = b"" with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: sock.connect((server, port)) - sock.sendall(f'{query}\n'.encode()) + sock.sendall(f"{query}\n".encode()) while True: data = sock.recv(2048) if not data: break bytes_whois += data - return { - 'results': [{'types': mispattributes['output'], 'values': bytes_whois.decode()}] - } + return {"results": [{"types": mispattributes["output"], "values": bytes_whois.decode()}]} def introspection(): @@ -68,5 +66,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/whoisfreaks.py b/misp_modules/modules/expansion/whoisfreaks.py index d64e21948..0d097b2a8 100644 --- a/misp_modules/modules/expansion/whoisfreaks.py +++ b/misp_modules/modules/expansion/whoisfreaks.py @@ -2,30 +2,47 @@ import requests -misperrors = {'error': 'Error'} +misperrors = {"error": "Error"} mispattributes = { - 'input': ['domain'], - 'output': ['domain', 'dns-soa-email', - 'whois-registrant-email', 'whois-registrant-phone', - 'whois-registrant-name', - 'whois-registrar', 'whois-creation-date', 'domain'] + "input": ["domain"], + "output": [ + "domain", + "dns-soa-email", + "whois-registrant-email", + "whois-registrant-phone", + "whois-registrant-name", + "whois-registrar", + "whois-creation-date", + "domain", + ], } moduleinfo = { - 'version': '1', - 'author': 'WhoisFreaks', - 'description': 'An expansion module for https://whoisfreaks.com/ that will provide an enriched analysis of the provided domain, including WHOIS and DNS information.', - 'module-type': ['expansion', 'hover'], - 'name': 'WhoisFreaks Lookup', - 'logo': 'whoisfreaks.png', - 'requirements': ['An access to the Whoisfreaks API_KEY'], - 'features': 'The module takes a domain as input and queries the Whoisfreaks API with it.\n\nSome parsing operations are then processed on the result of the query to extract as much information as possible.\n\nAfter this we map the extracted data to MISP attributes.', - 'references': ['https://whoisfreaks.com/'], - 'input': 'A domain whose Data is required', - 'output': 'MISP attributes resulting from the query on Whoisfreaks API, included in the following list:\n- domain\n- dns-soa-email\n- whois-registrant-email\n- whois-registrant-phone\n- whois-registrant-name\n- whois-registrar\n- whois-creation-date\n- domain', + "version": "1", + "author": "WhoisFreaks", + "description": ( + "An expansion module for https://whoisfreaks.com/ that will provide an enriched analysis of the provided" + " domain, including WHOIS and DNS information." + ), + "module-type": ["expansion", "hover"], + "name": "WhoisFreaks Lookup", + "logo": "whoisfreaks.png", + "requirements": ["An access to the Whoisfreaks API_KEY"], + "features": ( + "The module takes a domain as input and queries the Whoisfreaks API with it.\n\nSome parsing operations are" + " then processed on the result of the query to extract as much information as possible.\n\nAfter this we map" + " the extracted data to MISP attributes." + ), + "references": ["https://whoisfreaks.com/"], + "input": "A domain whose Data is required", + "output": ( + "MISP attributes resulting from the query on Whoisfreaks API, included in the following list:\n- domain\n-" + " dns-soa-email\n- whois-registrant-email\n- whois-registrant-phone\n- whois-registrant-name\n-" + " whois-registrar\n- whois-creation-date\n- domain" + ), } # config fields that your code expects from the site admin -moduleconfig = ['apikey'] +moduleconfig = ["apikey"] def handler(q=False): @@ -33,17 +50,17 @@ def handler(q=False): request = json.loads(q) - if 'config' not in request or (not (request['config'].get('apikey') or ('apiKey' in request['config']))): - misperrors['error'] = 'WhoisFreaks authentication is missing' + request + if "config" not in request or (not (request["config"].get("apikey") or ("apiKey" in request["config"]))): + misperrors["error"] = "WhoisFreaks authentication is missing" + request return misperrors - apiKey = request['config'].get('apikey') + apiKey = request["config"].get("apikey") - if request.get('domain'): - domain = request['domain'] + if request.get("domain"): + domain = request["domain"] return handle_domain(apiKey, domain, misperrors) else: - misperrors['error'] = "Unsupported attributes types" + misperrors["error"] = "Unsupported attributes types" return misperrors else: return False @@ -54,12 +71,12 @@ def handle_domain(apiKey, domain, errors): r, status_ok = expand_whois(apiKey, domain) if status_ok: if r: - result_filtered['results'].extend(r) + result_filtered["results"].extend(r) r, status_ok = expand_dns(apiKey, domain) if status_ok: if r: - result_filtered['results'].extend(r) + result_filtered["results"].extend(r) return result_filtered @@ -75,66 +92,59 @@ def expand_whois(apiKey, domain): if results: status_ok = True - if 'create_date' in results: + if "create_date" in results: r.append( { - 'types': ['whois-creation-date'], - 'values': [results['create_date']], - 'categories': ['Attribution'], - 'comment': 'Creation Date for %s by whoisFreaks' - % domain - + "types": ["whois-creation-date"], + "values": [results["create_date"]], + "categories": ["Attribution"], + "comment": "Creation Date for %s by whoisFreaks" % domain, } ) - if 'domain_registrar' in results: - if 'registrar_name' in results['domain_registrar']: + if "domain_registrar" in results: + if "registrar_name" in results["domain_registrar"]: r.append( { - 'types': ['whois-registrant-name'], - 'values': [results['domain_registrar']['registrar_name']], - 'categories': ['Attribution'], - 'comment': 'Whois information of %s by whoisFreaks' - % domain + "types": ["whois-registrant-name"], + "values": [results["domain_registrar"]["registrar_name"]], + "categories": ["Attribution"], + "comment": "Whois information of %s by whoisFreaks" % domain, } ) - if 'email_address' in results['domain_registrar']: + if "email_address" in results["domain_registrar"]: r.append( { - 'types': ['whois-registrant-email'], - 'values': [results['domain_registrar']['email_address']], - 'categories': ['Attribution'], - 'comment': 'Whois information of %s by whoisFreaks' - % domain + "types": ["whois-registrant-email"], + "values": [results["domain_registrar"]["email_address"]], + "categories": ["Attribution"], + "comment": "Whois information of %s by whoisFreaks" % domain, } ) - if 'phone_number' in results['domain_registrar']: + if "phone_number" in results["domain_registrar"]: r.append( { - 'types': ['whois-registrant-email'], - 'values': [results['domain_registrar']['phone_number']], - 'categories': ['Attribution'], - 'comment': 'Whois information of %s by whoisFreaks' - % domain + "types": ["whois-registrant-email"], + "values": [results["domain_registrar"]["phone_number"]], + "categories": ["Attribution"], + "comment": "Whois information of %s by whoisFreaks" % domain, } ) - if 'name_servers' in results: - ns_servers = results['name_servers'] + if "name_servers" in results: + ns_servers = results["name_servers"] r.append( { - 'types': ['domain'], - 'values': ns_servers, - 'categories': ['Attribution'], - 'comment': 'Name server for %s by whoisFreaks' - % domain - + "types": ["domain"], + "values": ns_servers, + "categories": ["Attribution"], + "comment": "Name server for %s by whoisFreaks" % domain, } ) except Exception: - misperrors['error'] = "Error while processing Whois Data" + misperrors["error"] = "Error while processing Whois Data" return [], False return r, status_ok @@ -154,65 +164,68 @@ def expand_dns(apiKey, domain): if results: status_ok = True - if 'dnsRecords' in results: - dns_records = results['dnsRecords'] + if "dnsRecords" in results: + dns_records = results["dnsRecords"] for record in dns_records: - if record['dnsType'] == 'A': - list_ipv4.append(record['address']) - elif record['dnsType'] == 'AAAA': - list_ipv6.append(record['address']) - elif record['dnsType'] == 'MX': - servers_mx.append(record['target']) - elif record['dnsType'] == 'SOA': - soa_hostnames.append(record['host']) + if record["dnsType"] == "A": + list_ipv4.append(record["address"]) + elif record["dnsType"] == "AAAA": + list_ipv6.append(record["address"]) + elif record["dnsType"] == "MX": + servers_mx.append(record["target"]) + elif record["dnsType"] == "SOA": + soa_hostnames.append(record["host"]) if list_ipv4: - r.append({'types': ['domain|ip'], - 'values': ['%s|%s' % (domain, ipv4) for ipv4 in - list_ipv4], - 'categories': ['Network activity'], - 'comment': 'ipv4 of %s ' % - domain - }) + r.append( + { + "types": ["domain|ip"], + "values": ["%s|%s" % (domain, ipv4) for ipv4 in list_ipv4], + "categories": ["Network activity"], + "comment": "ipv4 of %s " % domain, + } + ) if list_ipv6: - r.append({'types': ['domain|ip'], - 'values': ['%s|%s' % (domain, ipv6) for ipv6 in - list_ipv6], - 'categories': ['Network activity'], - 'comment': 'ipv6 of %s' % - domain - }) + r.append( + { + "types": ["domain|ip"], + "values": ["%s|%s" % (domain, ipv6) for ipv6 in list_ipv6], + "categories": ["Network activity"], + "comment": "ipv6 of %s" % domain, + } + ) if servers_mx: - r.append({'types': ['domain'], - 'values': servers_mx, - 'categories': ['Network activity'], - 'comment': 'mx of %s' % - domain - }) + r.append( + { + "types": ["domain"], + "values": servers_mx, + "categories": ["Network activity"], + "comment": "mx of %s" % domain, + } + ) if soa_hostnames: - r.append({'types': ['domain'], - 'values': soa_hostnames, - 'categories': ['Network activity'], - 'comment': 'soa hostname of %s' % - domain - }) - + r.append( + { + "types": ["domain"], + "values": soa_hostnames, + "categories": ["Network activity"], + "comment": "soa hostname of %s" % domain, + } + ) except Exception: - misperrors['error'] = "Error while processing Whois Data" + misperrors["error"] = "Error while processing Whois Data" return [], False return r, status_ok def get_whois_response(domain, apiKey): - query = requests.get( - f"https://api.whoisfreaks.com/v1.0/whois?apiKey={apiKey}&whois=live&domainName={domain}" - ) + query = requests.get(f"https://api.whoisfreaks.com/v1.0/whois?apiKey={apiKey}&whois=live&domainName={domain}") if query.status_code != 200 and query.status_code != 206: - return {'error': f'Error while querying whoisfreaks.com - {query.status_code}: {query.reason}'} + return {"error": f"Error while querying whoisfreaks.com - {query.status_code}: {query.reason}"} return query.json() @@ -221,12 +234,14 @@ def get_dns_response(domain, apiKey): f"https://api.whoisfreaks.com/v1.0/dns/live?apiKey={apiKey}&domainName={domain}&type=SOA,AAAA,A,MX" ) if query.status_code != 200 and query.status_code != 206: - return {'error': f'Error while querying whoisfreaks.com - {query.status_code}: {query.reason}'} + return {"error": f"Error while querying whoisfreaks.com - {query.status_code}: {query.reason}"} return query.json() + def introspection(): return mispattributes + def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/wiki.py b/misp_modules/modules/expansion/wiki.py index ebbf8228a..5986f9dc6 100755 --- a/misp_modules/modules/expansion/wiki.py +++ b/misp_modules/modules/expansion/wiki.py @@ -1,51 +1,60 @@ import json -from SPARQLWrapper import SPARQLWrapper, JSON -misperrors = {'error': 'Error'} -mispattributes = {'input': ['text'], 'output': ['text']} +from SPARQLWrapper import JSON, SPARQLWrapper + +misperrors = {"error": "Error"} +mispattributes = {"input": ["text"], "output": ["text"]} moduleinfo = { - 'version': '0.2', - 'author': 'Roman Graf', - 'description': 'An expansion hover module to extract information from Wikidata to have additional information about particular term for analysis.', - 'module-type': ['hover'], - 'name': 'Wikidata Lookup', - 'logo': 'wikidata.png', - 'requirements': ['SPARQLWrapper python library'], - 'features': 'This module takes a text attribute as input and queries the Wikidata API. If the text attribute is clear enough to define a specific term, the API returns a wikidata link in response.', - 'references': ['https://www.wikidata.org'], - 'input': 'Text attribute.', - 'output': 'Text attribute.', + "version": "0.2", + "author": "Roman Graf", + "description": ( + "An expansion hover module to extract information from Wikidata to have additional information about particular" + " term for analysis." + ), + "module-type": ["hover"], + "name": "Wikidata Lookup", + "logo": "wikidata.png", + "requirements": ["SPARQLWrapper python library"], + "features": ( + "This module takes a text attribute as input and queries the Wikidata API. If the text attribute is clear" + " enough to define a specific term, the API returns a wikidata link in response." + ), + "references": ["https://www.wikidata.org"], + "input": "Text attribute.", + "output": "Text attribute.", } moduleconfig = [] # sample query text 'Microsoft' should provide Wikidata link https://www.wikidata.org/wiki/Q2283 in response -wiki_api_url = 'https://query.wikidata.org/bigdata/namespace/wdq/sparql' +wiki_api_url = "https://query.wikidata.org/bigdata/namespace/wdq/sparql" def handler(q=False): if q is False: return False request = json.loads(q) - if not request.get('text'): - misperrors['error'] = 'Query text missing' + if not request.get("text"): + misperrors["error"] = "Query text missing" return misperrors - sparql = SPARQLWrapper(wiki_api_url, agent='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36') - query_string = \ - "SELECT ?item \n" \ - "WHERE { \n" \ - "?item rdfs:label\"" + request.get('text') + "\" @en \n" \ - "}\n" + sparql = SPARQLWrapper( + wiki_api_url, + agent=( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko)" + " Chrome/50.0.2661.102 Safari/537.36" + ), + ) + query_string = 'SELECT ?item \nWHERE { \n?item rdfs:label"' + request.get("text") + '" @en \n}\n' sparql.setQuery(query_string) sparql.setReturnFormat(JSON) results = sparql.query().convert() try: result = results["results"]["bindings"] - summary = result[0]["item"]["value"] if result else 'No additional data found on Wikidata' + summary = result[0]["item"]["value"] if result else "No additional data found on Wikidata" except Exception as e: - misperrors['error'] = 'wikidata API not accessible {}'.format(e) - return misperrors['error'] + misperrors["error"] = "wikidata API not accessible {}".format(e) + return misperrors["error"] - r = {'results': [{'types': mispattributes['output'], 'values': summary}]} + r = {"results": [{"types": mispattributes["output"], "values": summary}]} return r @@ -54,5 +63,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/xforceexchange.py b/misp_modules/modules/expansion/xforceexchange.py index 865e72f72..cae5a814a 100644 --- a/misp_modules/modules/expansion/xforceexchange.py +++ b/misp_modules/modules/expansion/xforceexchange.py @@ -1,38 +1,57 @@ -import requests import json import sys -from . import check_input_attribute, standard_error_message from collections import defaultdict + +import requests from pymisp import MISPAttribute, MISPEvent, MISPObject from requests.auth import HTTPBasicAuth -sys.path.append('./') +from . import check_input_attribute, standard_error_message -misperrors = {'error': 'Error'} -mispattributes = {'input': ['ip-src', 'ip-dst', 'vulnerability', 'md5', 'sha1', 'sha256', 'domain', 'hostname', 'url'], - 'output': ['ip-src', 'ip-dst', 'text', 'domain'], - 'format': 'misp_standard'} +sys.path.append("./") + +misperrors = {"error": "Error"} +mispattributes = { + "input": [ + "ip-src", + "ip-dst", + "vulnerability", + "md5", + "sha1", + "sha256", + "domain", + "hostname", + "url", + ], + "output": ["ip-src", "ip-dst", "text", "domain"], + "format": "misp_standard", +} # possible module-types: 'expansion', 'hover' or both moduleinfo = { - 'version': '2', - 'author': 'Joerg Stephan (@johest)', - 'description': 'An expansion module for IBM X-Force Exchange.', - 'module-type': ['expansion', 'hover'], - 'name': 'IBM X-Force Exchange Lookup', - 'logo': 'xforce.png', - 'requirements': ['An access to the X-Force API (apikey)'], - 'features': 'This module takes a MISP attribute as input to query the X-Force API. The API returns then additional information known in their threats data, that is mapped into MISP attributes.', - 'references': ['https://exchange.xforce.ibmcloud.com/'], - 'input': 'A MISP attribute included in the following list:\n- ip-src\n- ip-dst\n- vulnerability\n- md5\n- sha1\n- sha256', - 'output': 'MISP attributes mapped from the result of the query on X-Force Exchange.', + "version": "2", + "author": "Joerg Stephan (@johest)", + "description": "An expansion module for IBM X-Force Exchange.", + "module-type": ["expansion", "hover"], + "name": "IBM X-Force Exchange Lookup", + "logo": "xforce.png", + "requirements": ["An access to the X-Force API (apikey)"], + "features": ( + "This module takes a MISP attribute as input to query the X-Force API. The API returns then additional" + " information known in their threats data, that is mapped into MISP attributes." + ), + "references": ["https://exchange.xforce.ibmcloud.com/"], + "input": ( + "A MISP attribute included in the following list:\n- ip-src\n- ip-dst\n- vulnerability\n- md5\n- sha1\n- sha256" + ), + "output": "MISP attributes mapped from the result of the query on X-Force Exchange.", } # config fields that your code expects from the site admin moduleconfig = ["apikey", "apipassword"] -class XforceExchange(): +class XforceExchange: def __init__(self, attribute, apikey, apipassword): self.base_url = "https://api.xforce.ibmcloud.com" self.misp_event = MISPEvent() @@ -42,123 +61,130 @@ def __init__(self, attribute, apikey, apipassword): self._apipassword = apipassword self.result = {} self.objects = defaultdict(dict) - self.status_mapping = {403: "Access denied, please check if your authentication is valid and if you did not reach the limit of queries.", - 404: "No result found for your query."} + self.status_mapping = { + 403: ( + "Access denied, please check if your authentication is valid and if you did not reach the limit of" + " queries." + ), + 404: "No result found for your query.", + } def parse(self): - mapping = {'url': '_parse_url', 'vulnerability': '_parse_vulnerability'} - mapping.update(dict.fromkeys(('md5', 'sha1', 'sha256'), '_parse_hash')) - mapping.update(dict.fromkeys(('domain', 'hostname'), '_parse_dns')) - mapping.update(dict.fromkeys(('ip-src', 'ip-dst'), '_parse_ip')) + mapping = {"url": "_parse_url", "vulnerability": "_parse_vulnerability"} + mapping.update(dict.fromkeys(("md5", "sha1", "sha256"), "_parse_hash")) + mapping.update(dict.fromkeys(("domain", "hostname"), "_parse_dns")) + mapping.update(dict.fromkeys(("ip-src", "ip-dst"), "_parse_ip")) to_call = mapping[self.attribute.type] getattr(self, to_call)(self.attribute.value) def get_result(self): if not self.misp_event.objects: - if 'error' not in self.result: - self.result['error'] = "No additional data found on Xforce Exchange." + if "error" not in self.result: + self.result["error"] = "No additional data found on Xforce Exchange." return self.result self.misp_event.add_attribute(**self.attribute) event = json.loads(self.misp_event.to_json()) - result = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} - return {'results': result} + result = {key: event[key] for key in ("Attribute", "Object") if (key in event and event[key])} + return {"results": result} def _api_call(self, url): try: result = requests.get(url, auth=HTTPBasicAuth(self._apikey, self._apipassword)) except Exception as e: - self.result['error'] = e + self.result["error"] = e return status_code = result.status_code if status_code != 200: try: - self.result['error'] = self.status_mapping[status_code] + self.result["error"] = self.status_mapping[status_code] except KeyError: - self.result['error'] = 'An error with the API has occurred.' + self.result["error"] = "An error with the API has occurred." return return result.json() def _create_file(self, malware, relationship): - file_object = MISPObject('file') - for key, relation in zip(('filepath', 'md5'), ('filename', 'md5')): + file_object = MISPObject("file") + for key, relation in zip(("filepath", "md5"), ("filename", "md5")): file_object.add_attribute(relation, malware[key]) file_object.add_reference(self.attribute.uuid, relationship) return file_object def _create_url(self, malware): - url_object = MISPObject('url') - for key, relation in zip(('uri', 'domain'), ('url', 'domain')): + url_object = MISPObject("url") + for key, relation in zip(("uri", "domain"), ("url", "domain")): url_object.add_attribute(relation, malware[key]) - attributes = tuple(f'{attribute.object_relation}_{attribute.value}' for attribute in url_object.attributes) - if attributes in self.objects['url']: + attributes = tuple(f"{attribute.object_relation}_{attribute.value}" for attribute in url_object.attributes) + if attributes in self.objects["url"]: del url_object - return self.objects['url'][attributes] + return self.objects["url"][attributes] url_uuid = url_object.uuid self.misp_event.add_object(**url_object) - self.objects['url'][attributes] = url_uuid + self.objects["url"][attributes] = url_uuid return url_uuid def _fetch_types(self, value): - if self.attribute.type in ('ip-src', 'ip-dst'): - return 'ip', 'domain', self.attribute.value - return 'domain', 'ip', value + if self.attribute.type in ("ip-src", "ip-dst"): + return "ip", "domain", self.attribute.value + return "domain", "ip", value def _handle_file(self, malware, relationship): file_object = self._create_file(malware, relationship) - attributes = tuple(f'{attribute.object_relation}_{attribute.value}' for attribute in file_object.attributes) - if attributes in self.objects['file']: - self.objects['file'][attributes].add_reference(self._create_url(malware), 'dropped-by') + attributes = tuple(f"{attribute.object_relation}_{attribute.value}" for attribute in file_object.attributes) + if attributes in self.objects["file"]: + self.objects["file"][attributes].add_reference(self._create_url(malware), "dropped-by") del file_object return - file_object.add_reference(self._create_url(malware), 'dropped-by') - self.objects['file'][attributes] = file_object + file_object.add_reference(self._create_url(malware), "dropped-by") + self.objects["file"][attributes] = file_object self.misp_event.add_object(**file_object) def _parse_dns(self, value): - dns_result = self._api_call(f'{self.base_url}/resolve/{value}') - if dns_result.get('Passive') and dns_result['Passive'].get('records'): - itype, ftype, value = self._fetch_types(dns_result['Passive']['query']) - misp_object = MISPObject('domain-ip') + dns_result = self._api_call(f"{self.base_url}/resolve/{value}") + if dns_result.get("Passive") and dns_result["Passive"].get("records"): + itype, ftype, value = self._fetch_types(dns_result["Passive"]["query"]) + misp_object = MISPObject("domain-ip") misp_object.add_attribute(itype, value) - for record in dns_result['Passive']['records']: - misp_object.add_attribute(ftype, record['value']) - misp_object.add_reference(self.attribute.uuid, 'related-to') + for record in dns_result["Passive"]["records"]: + misp_object.add_attribute(ftype, record["value"]) + misp_object.add_reference(self.attribute.uuid, "related-to") self.misp_event.add_object(**misp_object) def _parse_hash(self, value): - malware_result = self._api_call(f'{self.base_url}/malware/{value}') - if malware_result and malware_result.get('malware'): - malware_report = malware_result['malware'] - for malware in malware_report.get('origins', {}).get('CnCServers', {}).get('rows', []): - self._handle_file(malware, 'related-to') + malware_result = self._api_call(f"{self.base_url}/malware/{value}") + if malware_result and malware_result.get("malware"): + malware_report = malware_result["malware"] + for malware in malware_report.get("origins", {}).get("CnCServers", {}).get("rows", []): + self._handle_file(malware, "related-to") def _parse_ip(self, value): self._parse_dns(value) - self._parse_malware(value, 'ipr') + self._parse_malware(value, "ipr") def _parse_malware(self, value, feature): - malware_result = self._api_call(f'{self.base_url}/{feature}/malware/{value}') - if malware_result and malware_result.get('malware'): - for malware in malware_result['malware']: - self._handle_file(malware, 'associated-with') + malware_result = self._api_call(f"{self.base_url}/{feature}/malware/{value}") + if malware_result and malware_result.get("malware"): + for malware in malware_result["malware"]: + self._handle_file(malware, "associated-with") def _parse_url(self, value): self._parse_dns(value) - self._parse_malware(value, 'url') + self._parse_malware(value, "url") def _parse_vulnerability(self, value): - vulnerability_result = self._api_call(f'{self.base_url}/vulnerabilities/search/{value}') + vulnerability_result = self._api_call(f"{self.base_url}/vulnerabilities/search/{value}") if vulnerability_result: for vulnerability in vulnerability_result: - misp_object = MISPObject('vulnerability') - for code in vulnerability['stdcode']: - misp_object.add_attribute('id', code) - for feature, relation in zip(('title', 'description', 'temporal_score'), - ('summary', 'description', 'cvss-score')): + misp_object = MISPObject("vulnerability") + for code in vulnerability["stdcode"]: + misp_object.add_attribute("id", code) + for feature, relation in zip( + ("title", "description", "temporal_score"), + ("summary", "description", "cvss-score"), + ): misp_object.add_attribute(relation, vulnerability[feature]) - for reference in vulnerability['references']: - misp_object.add_attribute('references', reference['link_target']) - misp_object.add_reference(self.attribute.uuid, 'related-to') + for reference in vulnerability["references"]: + misp_object.add_attribute("references", reference["link_target"]) + misp_object.add_reference(self.attribute.uuid, "related-to") self.misp_event.add_object(**misp_object) @@ -166,16 +192,16 @@ def handler(q=False): if q is False: return False request = json.loads(q) - if not request.get('config') or not (request['config'].get('apikey') and request['config'].get('apipassword')): - misperrors['error'] = 'An API authentication is required (key and password).' + if not request.get("config") or not (request["config"].get("apikey") and request["config"].get("apipassword")): + misperrors["error"] = "An API authentication is required (key and password)." return misperrors key = request["config"]["apikey"] - password = request['config']['apipassword'] - if not request.get('attribute') or not check_input_attribute(request['attribute']): - return {'error': f'{standard_error_message} which should contain at least a type, a value and an uuid.'} - if request['attribute']['type'] not in mispattributes['input']: - return {'error': 'Unsupported attribute type.'} - parser = XforceExchange(request['attribute'], key, password) + password = request["config"]["apipassword"] + if not request.get("attribute") or not check_input_attribute(request["attribute"]): + return {"error": f"{standard_error_message} which should contain at least a type, a value and an uuid."} + if request["attribute"]["type"] not in mispattributes["input"]: + return {"error": "Unsupported attribute type."} + parser = XforceExchange(request["attribute"], key, password) parser.parse() return parser.get_result() @@ -185,5 +211,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/xlsx_enrich.py b/misp_modules/modules/expansion/xlsx_enrich.py index 3d71beeb1..29af93351 100644 --- a/misp_modules/modules/expansion/xlsx_enrich.py +++ b/misp_modules/modules/expansion/xlsx_enrich.py @@ -1,24 +1,27 @@ -import json import binascii +import io +import json + import np import pandas -import io -misperrors = {'error': 'Error'} -mispattributes = {'input': ['attachment'], - 'output': ['freetext', 'text']} +misperrors = {"error": "Error"} +mispattributes = {"input": ["attachment"], "output": ["freetext", "text"]} moduleinfo = { - 'version': '0.1', - 'author': 'Sascha Rommelfangen', - 'description': 'Module to extract freetext from a .xlsx document.', - 'module-type': ['expansion'], - 'name': 'XLXS Enrich', - 'logo': 'xlsx.png', - 'requirements': ['pandas: Python library to perform data analysis, time series and statistics.'], - 'features': 'The module reads the text contained in a .xlsx document. The result is passed to the freetext import parser so IoCs can be extracted out of it.', - 'references': [], - 'input': 'Attachment attribute containing a .xlsx document.', - 'output': 'Text and freetext parsed from the document.', + "version": "0.1", + "author": "Sascha Rommelfangen", + "description": "Module to extract freetext from a .xlsx document.", + "module-type": ["expansion"], + "name": "XLXS Enrich", + "logo": "xlsx.png", + "requirements": ["pandas: Python library to perform data analysis, time series and statistics."], + "features": ( + "The module reads the text contained in a .xlsx document. The result is passed to the freetext import parser so" + " IoCs can be extracted out of it." + ), + "references": [], + "input": "Attachment attribute containing a .xlsx document.", + "output": "Text and freetext parsed from the document.", } moduleconfig = [] @@ -28,29 +31,41 @@ def handler(q=False): if q is False: return False q = json.loads(q) - filename = q['attachment'] + filename = q["attachment"] try: - xlsx_array = np.frombuffer(binascii.a2b_base64(q['data']), np.uint8) + xlsx_array = np.frombuffer(binascii.a2b_base64(q["data"]), np.uint8) except Exception as e: print(e) err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?" - misperrors['error'] = err + misperrors["error"] = err print(err) return misperrors xls_content = "" xls_file = io.BytesIO(xlsx_array) - pandas.set_option('display.max_colwidth', -1) + pandas.set_option("display.max_colwidth", -1) try: xls = pandas.read_excel(xls_file) xls_content = xls.to_string(max_rows=None) print(xls_content) - return {'results': [{'types': ['freetext'], 'values': xls_content, 'comment': ".xlsx-to-text from file " + filename}, - {'types': ['text'], 'values': xls_content, 'comment': ".xlsx-to-text from file " + filename}]} + return { + "results": [ + { + "types": ["freetext"], + "values": xls_content, + "comment": ".xlsx-to-text from file " + filename, + }, + { + "types": ["text"], + "values": xls_content, + "comment": ".xlsx-to-text from file " + filename, + }, + ] + } except Exception as e: print(e) err = "Couldn't analyze file as .xlsx. Error was: " + str(e) - misperrors['error'] = err + misperrors["error"] = err return misperrors @@ -59,5 +74,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/yara_query.py b/misp_modules/modules/expansion/yara_query.py index dc320478c..ed0fa8448 100644 --- a/misp_modules/modules/expansion/yara_query.py +++ b/misp_modules/modules/expansion/yara_query.py @@ -1,39 +1,65 @@ import json import re -try: - import yara -except (OSError, ImportError): - print("yara is missing, use 'pip3 install -I -r REQUIREMENTS' from the root of this repository to install it.") -misperrors = {'error': 'Error'} +import yara + +misperrors = {"error": "Error"} moduleinfo = { - 'version': '1', - 'author': 'Christian STUDER', - 'description': 'The module takes a hash attribute (md5, sha1, sha256, imphash) as input, and is returning a YARA rule from it.', - 'module-type': ['expansion', 'hover'], - 'name': 'YARA Rule Generator', - 'require_standard_format': True, - 'logo': 'yara.png', - 'requirements': ['yara-python python library'], - 'features': "The module takes a hash attribute (md5, sha1, sha256, imphash) as input, and is returning a YARA rule from it. This YARA rule is also validated using the same method as in 'yara_syntax_validator' module.\nBoth hover and expansion functionalities are supported with this module, where the hover part is displaying the resulting YARA rule and the expansion part allows you to add the rule as a new attribute, as usual with expansion modules.", - 'references': ['https://virustotal.github.io/yara/', 'https://github.com/virustotal/yara-python'], - 'input': 'MISP Hash attribute (md5, sha1, sha256, imphash, or any of the composite attribute with filename and one of the previous hash type).', - 'output': 'YARA rule.', + "version": "1", + "author": "Christian STUDER", + "description": ( + "The module takes a hash attribute (md5, sha1, sha256, imphash) as input, and is returning a YARA rule from it." + ), + "module-type": ["expansion", "hover"], + "name": "YARA Rule Generator", + "require_standard_format": True, + "logo": "yara.png", + "requirements": ["yara-python python library"], + "features": ( + "The module takes a hash attribute (md5, sha1, sha256, imphash) as input, and is returning a YARA rule from it." + " This YARA rule is also validated using the same method as in 'yara_syntax_validator' module.\nBoth hover and" + " expansion functionalities are supported with this module, where the hover part is displaying the resulting" + " YARA rule and the expansion part allows you to add the rule as a new attribute, as usual with expansion" + " modules." + ), + "references": [ + "https://virustotal.github.io/yara/", + "https://github.com/virustotal/yara-python", + ], + "input": ( + "MISP Hash attribute (md5, sha1, sha256, imphash, or any of the composite attribute with filename and one of" + " the previous hash type)." + ), + "output": "YARA rule.", } moduleconfig = [] -mispattributes = {'input': ['md5', 'sha1', 'sha256', 'filename|md5', 'filename|sha1', 'filename|sha256', 'imphash'], 'output': ['yara']} +mispattributes = { + "input": [ + "md5", + "sha1", + "sha256", + "filename|md5", + "filename|sha1", + "filename|sha256", + "imphash", + ], + "output": ["yara"], +} def extract_input_attribute(request): - for input_type in mispattributes['input']: + for input_type in mispattributes["input"]: if input_type in request: return input_type, request[input_type] def get_hash_condition(hashtype, hashvalue): hashvalue = hashvalue.lower() - required_module, params = ('pe', '()') if hashtype == 'imphash' else ('hash', '(0, filesize)') - return '{}.{}{} == "{}"'.format(required_module, hashtype, params, hashvalue), required_module + required_module, params = ("pe", "()") if hashtype == "imphash" else ("hash", "(0, filesize)") + return ( + '{}.{}{} == "{}"'.format(required_module, hashtype, params, hashvalue), + required_module, + ) def handler(q=False): @@ -42,23 +68,27 @@ def handler(q=False): request = json.loads(q) attribute = extract_input_attribute(request) if attribute is None: - return {'error': f'Wrong input type, please choose in the following: {", ".join(mispattributes["input"])}'} - uuid = request.pop('attribute_uuid') if 'attribute_uuid' in request else None + return {"error": f'Wrong input type, please choose in the following: {", ".join(mispattributes["input"])}'} + uuid = request.pop("attribute_uuid") if "attribute_uuid" in request else None attribute_type, value = attribute - if 'filename' in attribute_type: - _, attribute_type = attribute_type.split('|') - _, value = value.split('|') + if "filename" in attribute_type: + _, attribute_type = attribute_type.split("|") + _, value = value.split("|") condition, required_module = get_hash_condition(attribute_type, value) import_section = 'import "{}"'.format(required_module) - rule_start = '%s\r\nrule %s_%s {' % (import_section, attribute_type.upper(), re.sub(r'\W+', '_', uuid)) if uuid else '%s\r\nrule %s {' % (import_section, attribute_type.upper()) - condition = '\tcondition:\r\n\t\t{}'.format(condition) - rule = '\r\n'.join([rule_start, condition, '}']) + rule_start = ( + "%s\r\nrule %s_%s {" % (import_section, attribute_type.upper(), re.sub(r"\W+", "_", uuid)) + if uuid + else "%s\r\nrule %s {" % (import_section, attribute_type.upper()) + ) + condition = "\tcondition:\r\n\t\t{}".format(condition) + rule = "\r\n".join([rule_start, condition, "}"]) try: yara.compile(source=rule) except Exception as e: - misperrors['error'] = 'Syntax error: {}'.format(e) + misperrors["error"] = "Syntax error: {}".format(e) return misperrors - return {'results': [{'types': mispattributes['output'], 'values': rule}]} + return {"results": [{"types": mispattributes["output"], "values": rule}]} def introspection(): @@ -66,5 +96,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/yara_syntax_validator.py b/misp_modules/modules/expansion/yara_syntax_validator.py index ce2b13665..cf02dd1e2 100644 --- a/misp_modules/modules/expansion/yara_syntax_validator.py +++ b/misp_modules/modules/expansion/yara_syntax_validator.py @@ -1,23 +1,24 @@ import json -try: - import yara -except (OSError, ImportError): - print("yara is missing, use 'pip3 install -I -r REQUIREMENTS' from the root of this repository to install it.") -misperrors = {'error': 'Error'} -mispattributes = {'input': ['yara'], 'output': ['text']} +import yara + +misperrors = {"error": "Error"} +mispattributes = {"input": ["yara"], "output": ["text"]} moduleinfo = { - 'version': '0.1', - 'author': 'Dennis Rand', - 'description': 'An expansion hover module to perform a syntax check on if yara rules are valid or not.', - 'module-type': ['hover'], - 'name': 'YARA Syntax Validator', - 'logo': 'yara.png', - 'requirements': ['yara_python python library'], - 'features': 'This modules simply takes a YARA rule as input, and checks its syntax. It returns then a confirmation if the syntax is valid, otherwise the syntax error is displayed.', - 'references': ['http://virustotal.github.io/yara/'], - 'input': 'YARA rule attribute.', - 'output': 'Text to inform users if their rule is valid.', + "version": "0.1", + "author": "Dennis Rand", + "description": "An expansion hover module to perform a syntax check on if yara rules are valid or not.", + "module-type": ["hover"], + "name": "YARA Syntax Validator", + "logo": "yara.png", + "requirements": ["yara_python python library"], + "features": ( + "This modules simply takes a YARA rule as input, and checks its syntax. It returns then a confirmation if the" + " syntax is valid, otherwise the syntax error is displayed." + ), + "references": ["http://virustotal.github.io/yara/"], + "input": "YARA rule attribute.", + "output": "Text to inform users if their rule is valid.", } moduleconfig = [] @@ -26,17 +27,17 @@ def handler(q=False): if q is False: return False request = json.loads(q) - if not request.get('yara'): - misperrors['error'] = 'Yara rule missing' + if not request.get("yara"): + misperrors["error"] = "Yara rule missing" return misperrors try: - yara.compile(source=request.get('yara')) - summary = ("Syntax valid") + yara.compile(source=request.get("yara")) + summary = "Syntax valid" except Exception as e: - summary = ("Syntax error: " + str(e)) + summary = "Syntax error: " + str(e) - r = {'results': [{'types': mispattributes['output'], 'values': summary}]} + r = {"results": [{"types": mispattributes["output"], "values": summary}]} return r @@ -45,5 +46,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/yeti.py b/misp_modules/modules/expansion/yeti.py index 5bee5c96c..d065d31cd 100644 --- a/misp_modules/modules/expansion/yeti.py +++ b/misp_modules/modules/expansion/yeti.py @@ -1,41 +1,57 @@ import json import logging -try: - import pyeti -except ImportError: - print("pyeti module not installed.") - +import pyeti from pymisp import MISPEvent, MISPObject -misperrors = {'error': 'Error'} - -mispattributes = {'input': ['AS', 'ip-src', 'ip-dst', 'hostname', 'domain', 'sha256', 'sha1', 'md5', 'url'], - 'format': 'misp_standard' - } +misperrors = {"error": "Error"} + +mispattributes = { + "input": [ + "AS", + "ip-src", + "ip-dst", + "hostname", + "domain", + "sha256", + "sha1", + "md5", + "url", + ], + "format": "misp_standard", +} # possible module-types: 'expansion', 'hover' or both moduleinfo = { - 'version': '1', - 'author': 'Sebastien Larinier @sebdraven', - 'description': 'Module to process a query on Yeti.', - 'module-type': ['expansion', 'hover'], - 'name': 'Yeti Lookup', - 'logo': 'yeti.png', - 'requirements': ['pyeti', 'API key '], - 'features': 'This module add context and links between observables using yeti', - 'references': ['https://github.com/yeti-platform/yeti', 'https://github.com/sebdraven/pyeti'], - 'input': 'A domain, hostname,IP, sha256,sha1, md5, url of MISP attribute.', - 'output': 'MISP attributes and objects fetched from the Yeti instances.', + "version": "1", + "author": "Sebastien Larinier @sebdraven", + "description": "Module to process a query on Yeti.", + "module-type": ["expansion", "hover"], + "name": "Yeti Lookup", + "logo": "yeti.png", + "requirements": ["pyeti", "API key "], + "features": "This module add context and links between observables using yeti", + "references": [ + "https://github.com/yeti-platform/yeti", + "https://github.com/sebdraven/pyeti", + ], + "input": "A domain, hostname,IP, sha256,sha1, md5, url of MISP attribute.", + "output": "MISP attributes and objects fetched from the Yeti instances.", } -moduleconfig = ['apikey', 'url'] +moduleconfig = ["apikey", "url"] -class Yeti(): +class Yeti: def __init__(self, url, key, attribute): - self.misp_mapping = {'Ip': 'ip-dst', 'Domain': 'domain', 'Hostname': 'hostname', 'Url': 'url', - 'AutonomousSystem': 'AS', 'File': 'sha256'} + self.misp_mapping = { + "Ip": "ip-dst", + "Domain": "domain", + "Hostname": "hostname", + "Url": "url", + "AutonomousSystem": "AS", + "File": "sha256", + } self.yeti_client = pyeti.YetiApi(url=url, api_key=key) self.attribute = attribute self.misp_event = MISPEvent() @@ -48,19 +64,27 @@ def search(self, value): def get_neighboors(self, obs_id): neighboors = self.yeti_client.neighbors_observables(obs_id) - if neighboors and 'objs' in neighboors: - links_by_id = {link['dst']['id']: (link['description'], 'dst') for link in neighboors['links'] - if link['dst']['id'] != obs_id} - links_by_id.update({link['src']['id']: (link['description'], 'src') for link in neighboors['links'] - if link['src']['id'] != obs_id}) - - for n in neighboors['objs']: - yield n, links_by_id[n['id']] + if neighboors and "objs" in neighboors: + links_by_id = { + link["dst"]["id"]: (link["description"], "dst") + for link in neighboors["links"] + if link["dst"]["id"] != obs_id + } + links_by_id.update( + { + link["src"]["id"]: (link["description"], "src") + for link in neighboors["links"] + if link["src"]["id"] != obs_id + } + ) + + for n in neighboors["objs"]: + yield n, links_by_id[n["id"]] def parse_yeti_result(self): - obs = self.search(self.attribute['value']) + obs = self.search(self.attribute["value"]) - for obs_to_add, link in self.get_neighboors(obs['id']): + for obs_to_add, link in self.get_neighboors(obs["id"]): object_misp_domain_ip = self.__get_object_domain_ip(obs_to_add) if object_misp_domain_ip: self.misp_event.add_object(object_misp_domain_ip) @@ -69,7 +93,7 @@ def parse_yeti_result(self): if object_misp_url: self.misp_event.add_object(object_misp_url) continue - if link[0] == 'NS record': + if link[0] == "NS record": object_ns_record = self.__get_object_ns_record(obs_to_add, link[1]) if object_ns_record: self.misp_event.add_object(object_ns_record) @@ -78,84 +102,85 @@ def parse_yeti_result(self): def get_result(self): event = json.loads(self.misp_event.to_json()) - results = {key: event[key] for key in ('Attribute', 'Object') if key in event} + results = {key: event[key] for key in ("Attribute", "Object") if key in event} return results def __get_attribute(self, obs_to_add, link): try: - type_attr = self.misp_mapping[obs_to_add['type']] + type_attr = self.misp_mapping[obs_to_add["type"]] value = None - if obs_to_add['type'] == 'File': - value = obs_to_add['value'].split(':')[1] + if obs_to_add["type"] == "File": + value = obs_to_add["value"].split(":")[1] else: - value = obs_to_add['value'] + value = obs_to_add["value"] attr = self.misp_event.add_attribute(value=value, type=type_attr) - attr.comment = '%s: %s' % (link, self.attribute['value']) + attr.comment = "%s: %s" % (link, self.attribute["value"]) except KeyError: - logging.error('type not found %s' % obs_to_add['type']) + logging.error("type not found %s" % obs_to_add["type"]) return - for t in obs_to_add['tags']: - self.misp_event.add_attribute_tag(t['name'], attr['uuid']) + for t in obs_to_add["tags"]: + self.misp_event.add_attribute_tag(t["name"], attr["uuid"]) def __get_object_domain_ip(self, obj_to_add): - if (obj_to_add['type'] == 'Ip' and self.attribute['type'] in ['hostname', 'domain']) or \ - (obj_to_add['type'] in ('Hostname', 'Domain') and self.attribute['type'] in ('ip-src', 'ip-dst')): - domain_ip_object = MISPObject('domain-ip') - domain_ip_object.add_attribute(self.__get_relation(obj_to_add), - obj_to_add['value']) - domain_ip_object.add_attribute(self.__get_relation(self.attribute, is_yeti_object=False), - self.attribute['value']) - domain_ip_object.add_reference(self.attribute['uuid'], 'related_to') + if (obj_to_add["type"] == "Ip" and self.attribute["type"] in ["hostname", "domain"]) or ( + obj_to_add["type"] in ("Hostname", "Domain") and self.attribute["type"] in ("ip-src", "ip-dst") + ): + domain_ip_object = MISPObject("domain-ip") + domain_ip_object.add_attribute(self.__get_relation(obj_to_add), obj_to_add["value"]) + domain_ip_object.add_attribute( + self.__get_relation(self.attribute, is_yeti_object=False), + self.attribute["value"], + ) + domain_ip_object.add_reference(self.attribute["uuid"], "related_to") return domain_ip_object def __get_object_url(self, obj_to_add): - if (obj_to_add['type'] == 'Url' and self.attribute['type'] in ['hostname', 'domain', 'ip-src', 'ip-dst']) or ( - obj_to_add['type'] in ('Hostname', 'Domain', 'Ip') and self.attribute['type'] == 'url' + if (obj_to_add["type"] == "Url" and self.attribute["type"] in ["hostname", "domain", "ip-src", "ip-dst"]) or ( + obj_to_add["type"] in ("Hostname", "Domain", "Ip") and self.attribute["type"] == "url" ): - url_object = MISPObject('url') + url_object = MISPObject("url") obj_relation = self.__get_relation(obj_to_add) if obj_relation: - url_object.add_attribute(obj_relation, obj_to_add['value']) + url_object.add_attribute(obj_relation, obj_to_add["value"]) obj_relation = self.__get_relation(self.attribute, is_yeti_object=False) if obj_relation: - url_object.add_attribute(obj_relation, - self.attribute['value']) - url_object.add_reference(self.attribute['uuid'], 'related_to') + url_object.add_attribute(obj_relation, self.attribute["value"]) + url_object.add_reference(self.attribute["uuid"], "related_to") return url_object def __get_object_ns_record(self, obj_to_add, link): queried_domain = None ns_domain = None - object_dns_record = MISPObject('dns-record') - if link == 'dst': - queried_domain = self.attribute['value'] - ns_domain = obj_to_add['value'] - elif link == 'src': - queried_domain = obj_to_add['value'] - ns_domain = self.attribute['value'] + object_dns_record = MISPObject("dns-record") + if link == "dst": + queried_domain = self.attribute["value"] + ns_domain = obj_to_add["value"] + elif link == "src": + queried_domain = obj_to_add["value"] + ns_domain = self.attribute["value"] if queried_domain and ns_domain: - object_dns_record.add_attribute('queried-domain', queried_domain) - object_dns_record.add_attribute('ns-record', ns_domain) - object_dns_record.add_reference(self.attribute['uuid'], 'related_to') + object_dns_record.add_attribute("queried-domain", queried_domain) + object_dns_record.add_attribute("ns-record", ns_domain) + object_dns_record.add_reference(self.attribute["uuid"], "related_to") return object_dns_record def __get_relation(self, obj, is_yeti_object=True): if is_yeti_object: - type_attribute = self.misp_mapping[obj['type']] + type_attribute = self.misp_mapping[obj["type"]] else: - type_attribute = obj['type'] - if type_attribute == 'ip-src' or type_attribute == 'ip-dst': - return 'ip' - elif 'domain' == type_attribute: - return 'domain' - elif 'hostname' == type_attribute: - return 'domain' - elif type_attribute == 'url': + type_attribute = obj["type"] + if type_attribute == "ip-src" or type_attribute == "ip-dst": + return "ip" + elif "domain" == type_attribute: + return "domain" + elif "hostname" == type_attribute: + return "domain" + elif type_attribute == "url": return type_attribute @@ -168,27 +193,27 @@ def handler(q=False): yeti_client = None request = json.loads(q) - attribute = request['attribute'] - if attribute['type'] not in mispattributes['input']: - return {'error': 'Unsupported attributes type'} - - if 'config' in request and 'url' in request['config']: - yeti_url = request['config']['url'] - if 'config' in request and 'apikey' in request['config']: - apikey = request['config']['apikey'] + attribute = request["attribute"] + if attribute["type"] not in mispattributes["input"]: + return {"error": "Unsupported attributes type"} + + if "config" in request and "url" in request["config"]: + yeti_url = request["config"]["url"] + if "config" in request and "apikey" in request["config"]: + apikey = request["config"]["apikey"] if apikey and yeti_url: yeti_client = Yeti(yeti_url, apikey, attribute) if yeti_client: yeti_client.parse_yeti_result() - return {'results': yeti_client.get_result()} + return {"results": yeti_client.get_result()} else: - misperrors['error'] = 'Yeti Config Error' + misperrors["error"] = "Yeti Config Error" return misperrors def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/export_mod/__init__.py b/misp_modules/modules/export_mod/__init__.py index d865527d2..e69de29bb 100644 --- a/misp_modules/modules/export_mod/__init__.py +++ b/misp_modules/modules/export_mod/__init__.py @@ -1,3 +0,0 @@ -__all__ = ['cef_export', 'mass_eql_export', 'liteexport', 'goamlexport', 'threat_connect_export', 'pdfexport', - 'threatStream_misp_export', 'osqueryexport', 'nexthinkexport', 'vt_graph', 'defender_endpoint_export', - 'virustotal_collections', 'yara_export', 'cisco_firesight_manager_ACL_rule_export'] diff --git a/misp_modules/modules/export_mod/cef_export.py b/misp_modules/modules/export_mod/cef_export.py index 2e57e773c..cf33215c7 100755 --- a/misp_modules/modules/export_mod/cef_export.py +++ b/misp_modules/modules/export_mod/cef_export.py @@ -1,32 +1,47 @@ -import json import base64 import datetime +import json -misperrors = {'error': 'Error'} +misperrors = {"error": "Error"} # possible module-types: 'expansion', 'hover' or both moduleinfo = { - 'version': '1', - 'author': 'Hannah Ward', - 'description': 'Module to export a MISP event in CEF format.', - 'module-type': ['export'], - 'name': 'CEF Export', - 'logo': '', - 'requirements': [], - 'features': 'The module takes a MISP event in input, to look every attribute. Each attribute matching with some predefined types is then exported in Common Event Format.\nThus, there is no particular feature concerning MISP Events since any event can be exported. However, 4 configuration parameters recognized by CEF format are required and should be provided by users before exporting data: the device vendor, product and version, as well as the default severity of data.', - 'references': ['https://community.softwaregrp.com/t5/ArcSight-Connectors/ArcSight-Common-Event-Format-CEF-Guide/ta-p/1589306?attachment-id=65537'], - 'input': 'MISP Event attributes', - 'output': 'Common Event Format file', + "version": "1", + "author": "Hannah Ward", + "description": "Module to export a MISP event in CEF format.", + "module-type": ["export"], + "name": "CEF Export", + "logo": "", + "requirements": [], + "features": ( + "The module takes a MISP event in input, to look every attribute. Each attribute matching with some predefined" + " types is then exported in Common Event Format.\nThus, there is no particular feature concerning MISP Events" + " since any event can be exported. However, 4 configuration parameters recognized by CEF format are required" + " and should be provided by users before exporting data: the device vendor, product and version, as well as the" + " default severity of data." + ), + "references": [ + "https://community.softwaregrp.com/t5/ArcSight-Connectors/ArcSight-Common-Event-Format-CEF-Guide/ta-p/1589306?attachment-id=65537" + ], + "input": "MISP Event attributes", + "output": "Common Event Format file", } # config fields that your code expects from the site admin moduleconfig = ["Default_Severity", "Device_Vendor", "Device_Product", "Device_Version"] -cefmapping = {"ip-src": "src", "ip-dst": "dst", "hostname": "dhost", "domain": "dhost", - "md5": "fileHash", "sha1": "fileHash", "sha256": "fileHash", - "url": "request"} +cefmapping = { + "ip-src": "src", + "ip-dst": "dst", + "hostname": "dhost", + "domain": "dhost", + "md5": "fileHash", + "sha1": "fileHash", + "sha256": "fileHash", + "url": "request", +} -mispattributes = {'input': list(cefmapping.keys())} +mispattributes = {"input": list(cefmapping.keys())} outputFileExtension = "cef" responseType = "application/txt" @@ -38,8 +53,12 @@ def handler(q=False): if "config" in request: config = request["config"] else: - config = {"Default_Severity": 1, "Device_Vendor": "MISP", - "Device_Product": "MISP", "Device_Version": 1} + config = { + "Default_Severity": 1, + "Device_Vendor": "MISP", + "Device_Product": "MISP", + "Device_Version": 1, + } data = request["data"] response = "" @@ -48,18 +67,21 @@ def handler(q=False): for attr in event: if attr["type"] in cefmapping: response += "{} host CEF:0|{}|{}|{}|{}|{}|{}|{}={}\n".format( - datetime.datetime.fromtimestamp(int(attr["timestamp"])).strftime("%b %d %H:%M:%S"), - config["Device_Vendor"], - config["Device_Product"], - config["Device_Version"], - attr["category"], - attr["category"], - config["Default_Severity"], - cefmapping[attr["type"]], - attr["value"], + datetime.datetime.fromtimestamp(int(attr["timestamp"])).strftime("%b %d %H:%M:%S"), + config["Device_Vendor"], + config["Device_Product"], + config["Device_Version"], + attr["category"], + attr["category"], + config["Default_Severity"], + cefmapping[attr["type"]], + attr["value"], ) - r = {"response": [], "data": str(base64.b64encode(bytes(response, 'utf-8')), 'utf-8')} + r = { + "response": [], + "data": str(base64.b64encode(bytes(response, "utf-8")), "utf-8"), + } return r @@ -67,27 +89,27 @@ def introspection(): modulesetup = {} try: responseType - modulesetup['responseType'] = responseType + modulesetup["responseType"] = responseType except NameError: pass try: userConfig - modulesetup['userConfig'] = userConfig + modulesetup["userConfig"] = userConfig except NameError: pass try: outputFileExtension - modulesetup['outputFileExtension'] = outputFileExtension + modulesetup["outputFileExtension"] = outputFileExtension except NameError: pass try: inputSource - modulesetup['inputSource'] = inputSource + modulesetup["inputSource"] = inputSource except NameError: pass return modulesetup def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py b/misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py index fcc7e4da9..020f240aa 100644 --- a/misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py +++ b/misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py @@ -9,38 +9,41 @@ # # ###################################################### -import json import base64 +import json from urllib.parse import quote -misperrors = {'error': 'Error'} +misperrors = {"error": "Error"} moduleinfo = { - 'version': '1', - 'author': 'Stanislav Klevtsov', - 'description': 'Module to export malicious network activity attributes to Cisco fireSIGHT manager block rules.', - 'module-type': ['export'], - 'name': 'Cisco fireSIGHT blockrule Export', - 'logo': 'cisco.png', - 'requirements': ['Firesight manager console credentials'], - 'features': 'The module goes through the attributes to find all the network activity ones in order to create block rules for the Cisco fireSIGHT manager.', - 'references': [], - 'input': 'Network activity attributes (IPs, URLs).', - 'output': 'Cisco fireSIGHT manager block rules.', + "version": "1", + "author": "Stanislav Klevtsov", + "description": "Module to export malicious network activity attributes to Cisco fireSIGHT manager block rules.", + "module-type": ["export"], + "name": "Cisco fireSIGHT blockrule Export", + "logo": "cisco.png", + "requirements": ["Firesight manager console credentials"], + "features": ( + "The module goes through the attributes to find all the network activity ones in order to create block rules" + " for the Cisco fireSIGHT manager." + ), + "references": [], + "input": "Network activity attributes (IPs, URLs).", + "output": "Cisco fireSIGHT manager block rules.", } -moduleconfig = ['fmc_ip_addr', 'fmc_login', 'fmc_pass', 'domain_id', 'acpolicy_id'] +moduleconfig = ["fmc_ip_addr", "fmc_login", "fmc_pass", "domain_id", "acpolicy_id"] fsmapping = {"ip-dst": "dst", "url": "request"} -mispattributes = {'input': list(fsmapping.keys())} +mispattributes = {"input": list(fsmapping.keys())} # options: event, attribute, event-collection, attribute-collection -inputSource = ['event'] +inputSource = ["event"] -outputFileExtension = 'sh' -responseType = 'application/txt' +outputFileExtension = "sh" +responseType = "application/txt" # .sh file templates SH_FILE_HEADER = """#!/bin/sh\n\n""" @@ -60,23 +63,23 @@ def handler(q=False): if q is False: return False - r = {'results': []} + r = {"results": []} request = json.loads(q) if "config" in request: config = request["config"] # check if config is empty - if not config['fmc_ip_addr']: - config['fmc_ip_addr'] = "0.0.0.0" - if not config['fmc_login']: - config['fmc_login'] = "login" - if not config['fmc_pass']: - config['fmc_pass'] = "password" - if not config['domain_id']: - config['domain_id'] = "SET_FIRESIGHT_DOMAIN_ID" - if not config['acpolicy_id']: - config['acpolicy_id'] = "SET_FIRESIGHT_ACPOLICY_ID" + if not config["fmc_ip_addr"]: + config["fmc_ip_addr"] = "0.0.0.0" + if not config["fmc_login"]: + config["fmc_login"] = "login" + if not config["fmc_pass"]: + config["fmc_pass"] = "password" + if not config["domain_id"]: + config["domain_id"] = "SET_FIRESIGHT_DOMAIN_ID" + if not config["acpolicy_id"]: + config["acpolicy_id"] = "SET_FIRESIGHT_ACPOLICY_ID" data = request["data"] output = "" @@ -96,27 +99,36 @@ def handler(q=False): if attr["type"] == "ip-dst": ipdst.append(BLOCK_DST_JSON_TMPL.format(ipdst=attr["value"])) else: - urls.append(BLOCK_URL_JSON_TMPL.format(url=quote(attr["value"], safe='@/:;?&=-_.,+!*'))) + urls.append(BLOCK_URL_JSON_TMPL.format(url=quote(attr["value"], safe="@/:;?&=-_.,+!*"))) # building the .sh file output += SH_FILE_HEADER - output += "FIRESIGHT_IP_ADDR='{}'\n".format(config['fmc_ip_addr']) - - output += "LOGINPASS_BASE64=`echo -n '{}:{}' | base64`\n".format(config['fmc_login'], config['fmc_pass']) - output += "DOMAIN_ID='{}'\n".format(config['domain_id']) - output += "ACPOLICY_ID='{}'\n\n".format(config['acpolicy_id']) - - output += "ACC_TOKEN=`curl -X POST -v -k -sD - -o /dev/null -H \"Authorization: Basic $LOGINPASS_BASE64\" -i \"https://$FIRESIGHT_IP_ADDR/api/fmc_platform/v1/auth/generatetoken\" | grep -i x-auth-acc | sed 's/.*:\\ //g' | tr -d '[:space:]' | tr -d '\\n'`\n" - - output += BLOCK_JSON_TMPL.format(rule_name="misp_event_{}".format(event_id), - dst_networks=', '.join(ipdst), - urls=', '.join(urls), - event_info_comment=event_info) + "\n" + output += "FIRESIGHT_IP_ADDR='{}'\n".format(config["fmc_ip_addr"]) + + output += "LOGINPASS_BASE64=`echo -n '{}:{}' | base64`\n".format(config["fmc_login"], config["fmc_pass"]) + output += "DOMAIN_ID='{}'\n".format(config["domain_id"]) + output += "ACPOLICY_ID='{}'\n\n".format(config["acpolicy_id"]) + + output += ( + 'ACC_TOKEN=`curl -X POST -v -k -sD - -o /dev/null -H "Authorization: Basic $LOGINPASS_BASE64" -i' + ' "https://$FIRESIGHT_IP_ADDR/api/fmc_platform/v1/auth/generatetoken" | grep -i x-auth-acc | sed \'s/.*:\\' + " //g' | tr -d '[:space:]' | tr -d '\\n'`\n" + ) + + output += ( + BLOCK_JSON_TMPL.format( + rule_name="misp_event_{}".format(event_id), + dst_networks=", ".join(ipdst), + urls=", ".join(urls), + event_info_comment=event_info, + ) + + "\n" + ) output += CURL_ADD_RULE_TMPL # END building the .sh file - r = {"data": base64.b64encode(output.encode('utf-8')).decode('utf-8')} + r = {"data": base64.b64encode(output.encode("utf-8")).decode("utf-8")} return r @@ -124,27 +136,27 @@ def introspection(): modulesetup = {} try: responseType - modulesetup['responseType'] = responseType + modulesetup["responseType"] = responseType except NameError: pass try: userConfig - modulesetup['userConfig'] = userConfig + modulesetup["userConfig"] = userConfig except NameError: pass try: outputFileExtension - modulesetup['outputFileExtension'] = outputFileExtension + modulesetup["outputFileExtension"] = outputFileExtension except NameError: pass try: inputSource - modulesetup['inputSource'] = inputSource + modulesetup["inputSource"] = inputSource except NameError: pass return modulesetup def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/export_mod/defender_endpoint_export.py b/misp_modules/modules/export_mod/defender_endpoint_export.py index c283cb43d..92974f9ae 100755 --- a/misp_modules/modules/export_mod/defender_endpoint_export.py +++ b/misp_modules/modules/export_mod/defender_endpoint_export.py @@ -8,30 +8,34 @@ misperrors = {"error": "Error"} -types_to_use = ['sha256', 'sha1', 'md5', 'domain', 'ip-src', 'ip-dst', 'url'] +types_to_use = ["sha256", "sha1", "md5", "domain", "ip-src", "ip-dst", "url"] -userConfig = { - -} +userConfig = {} moduleconfig = ["Period"] -inputSource = ['event'] +inputSource = ["event"] -outputFileExtension = 'kql' -responseType = 'application/txt' +outputFileExtension = "kql" +responseType = "application/txt" moduleinfo = { - 'version': '1.1', - 'author': 'Julien Bachmann, Hacknowledge, Maik Wuerth', - 'description': 'Defender for Endpoint KQL hunting query export module', - 'module-type': ['export'], - 'name': 'Microsoft Defender for Endpoint KQL Export', - 'logo': 'defender_endpoint.png', - 'requirements': [], - 'features': 'This module export an event as Defender for Endpoint KQL queries that can then be used in your own python3 or Powershell tool. If you are using Microsoft Sentinel, you can directly connect your MISP instance to Sentinel and then create queries using the `ThreatIntelligenceIndicator` table to match events against imported IOC.', - 'references': ['https://docs.microsoft.com/en-us/windows/security/threat-protection/microsoft-defender-atp/advanced-hunting-schema-reference'], - 'input': 'MISP Event attributes', - 'output': 'Defender for Endpoint KQL queries', + "version": "1.1", + "author": "Julien Bachmann, Hacknowledge, Maik Wuerth", + "description": "Defender for Endpoint KQL hunting query export module", + "module-type": ["export"], + "name": "Microsoft Defender for Endpoint KQL Export", + "logo": "defender_endpoint.png", + "requirements": [], + "features": ( + "This module export an event as Defender for Endpoint KQL queries that can then be used in your own python3 or" + " Powershell tool. If you are using Microsoft Sentinel, you can directly connect your MISP instance to Sentinel" + " and then create queries using the `ThreatIntelligenceIndicator` table to match events against imported IOC." + ), + "references": [ + "https://docs.microsoft.com/en-us/windows/security/threat-protection/microsoft-defender-atp/advanced-hunting-schema-reference" + ], + "input": "MISP Event attributes", + "output": "Defender for Endpoint KQL queries", } @@ -39,35 +43,35 @@ def handle_sha256(value, period): query = f"""find in (DeviceEvents, DeviceAlertEvents,AlertInfo, AlertEvidence, DeviceFileEvents, DeviceImageLoadEvents, DeviceProcessEvents) where (SHA256 == '{value}' or InitiatingProcessSHA1 == '{value}') and Timestamp between(ago({period}) .. now())""" - return query.replace('\n', ' ') + return query.replace("\n", " ") def handle_sha1(value, period): query = f"""find in (DeviceEvents, DeviceAlertEvents, AlertInfo, AlertEvidence, DeviceFileEvents, DeviceImageLoadEvents, DeviceProcessEvents) where (SHA1 == '{value}' or InitiatingProcessSHA1 == '{value}') and Timestamp between(ago({period}) .. now())""" - return query.replace('\n', ' ') + return query.replace("\n", " ") def handle_md5(value, period): query = f"""find in (DeviceEvents, DeviceAlertEvents, AlertInfo, AlertEvidence, DeviceFileEvents, DeviceImageLoadEvents, DeviceProcessEvents) where (MD5 == '{value}' or InitiatingProcessMD5 == '{value}') and Timestamp between(ago({period}) .. now())""" - return query.replace('\n', ' ') + return query.replace("\n", " ") def handle_domain(value, period): query = f"""find in (DeviceAlertEvents, AlertInfo, AlertEvidence, DeviceNetworkEvents) where RemoteUrl contains '{value}' and Timestamp between(ago({period}) .. now())""" - return query.replace('\n', ' ') + return query.replace("\n", " ") def handle_ip(value, period): query = f"""find in (DeviceAlertEvents, AlertInfo, AlertEvidence, DeviceNetworkEvents) where RemoteIP == '{value}' and Timestamp between(ago({period}) .. now())""" - return query.replace('\n', ' ') + return query.replace("\n", " ") def handle_url(value, period): @@ -78,17 +82,17 @@ def handle_url(value, period): or FileOriginUrl has url or FileOriginReferrerUrl has url or Url has url""" - return query.replace('\n', ' ') + return query.replace("\n", " ") handlers = { - 'sha256': handle_sha256, - 'sha1': handle_sha1, - 'md5': handle_md5, - 'domain': handle_url, - 'ip-src': handle_ip, - 'ip-dst': handle_ip, - 'url': handle_url + "sha256": handle_sha256, + "sha1": handle_sha1, + "md5": handle_md5, + "domain": handle_url, + "ip-src": handle_ip, + "ip-dst": handle_ip, + "url": handle_url, } @@ -97,17 +101,17 @@ def handler(q=False): return False request = json.loads(q) config = request.get("config", {"Period": ""}) - output = '' + output = "" for event in request["data"]: for attribute in event["Attribute"]: - if attribute['type'] in types_to_use: - output = output + handlers[attribute['type']](attribute['value'], config['Period']) + '\n' + if attribute["type"] in types_to_use: + output = output + handlers[attribute["type"]](attribute["value"], config["Period"]) + "\n" for obj in event["Object"]: for attribute in obj["Attribute"]: - if attribute['type'] in types_to_use: - output = output + handlers[attribute['type']](attribute['value'], config['Period']) + '\n' - r = {"response": [], "data": str(base64.b64encode(bytes(output, 'utf-8')), 'utf-8')} + if attribute["type"] in types_to_use: + output = output + handlers[attribute["type"]](attribute["value"], config["Period"]) + "\n" + r = {"response": [], "data": str(base64.b64encode(bytes(output, "utf-8")), "utf-8")} return r @@ -115,27 +119,27 @@ def introspection(): modulesetup = {} try: responseType - modulesetup['responseType'] = responseType + modulesetup["responseType"] = responseType except NameError: pass try: userConfig - modulesetup['userConfig'] = userConfig + modulesetup["userConfig"] = userConfig except NameError: pass try: outputFileExtension - modulesetup['outputFileExtension'] = outputFileExtension + modulesetup["outputFileExtension"] = outputFileExtension except NameError: pass try: inputSource - modulesetup['inputSource'] = inputSource + modulesetup["inputSource"] = inputSource except NameError: pass return modulesetup def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/export_mod/goamlexport.py b/misp_modules/modules/export_mod/goamlexport.py index a1ffaf911..ebac8c242 100644 --- a/misp_modules/modules/export_mod/goamlexport.py +++ b/misp_modules/modules/export_mod/goamlexport.py @@ -1,62 +1,135 @@ -import json import base64 +import json +from collections import Counter, defaultdict + from pymisp import MISPEvent -from collections import defaultdict, Counter -misperrors = {'error': 'Error'} +misperrors = {"error": "Error"} moduleinfo = { - 'version': '1', - 'author': 'Christian Studer', - 'description': 'This module is used to export MISP events containing transaction objects into GoAML format.', - 'module-type': ['export'], - 'name': 'GoAML Export', - 'require_standard_format': True, - 'logo': 'goAML.jpg', - 'requirements': ['PyMISP', 'MISP objects'], - 'features': "The module works as long as there is at least one transaction object in the Event.\n\nThen in order to have a valid GoAML document, please follow these guidelines:\n- For each transaction object, use either a bank-account, person, or legal-entity object to describe the origin of the transaction, and again one of them to describe the target of the transaction.\n- Create an object reference for both origin and target objects of the transaction.\n- A bank-account object needs a signatory, which is a person object, put as object reference of the bank-account.\n- A person can have an address, which is a geolocation object, put as object reference of the person.\n\nSupported relation types for object references that are recommended for each object are the folowing:\n- transaction:\n\t- 'from', 'from_my_client': Origin of the transaction - at least one of them is required.\n\t- 'to', 'to_my_client': Target of the transaction - at least one of them is required.\n\t- 'address': Location of the transaction - optional.\n- bank-account:\n\t- 'signatory': Signatory of a bank-account - the reference from bank-account to a signatory is required, but the relation-type is optional at the moment since this reference will always describe a signatory.\n\t- 'entity': Entity owning the bank account - optional.\n- person:\n\t- 'address': Address of a person - optional.", - 'references': ['http://goaml.unodc.org/'], - 'input': 'MISP objects (transaction, bank-account, person, legal-entity, geolocation), with references, describing financial transactions and their origin and target.', - 'output': 'GoAML format file, describing financial transactions, with their origin and target (bank accounts, persons or entities).', + "version": "1", + "author": "Christian Studer", + "description": "This module is used to export MISP events containing transaction objects into GoAML format.", + "module-type": ["export"], + "name": "GoAML Export", + "require_standard_format": True, + "logo": "goAML.jpg", + "requirements": ["PyMISP", "MISP objects"], + "features": ( + "The module works as long as there is at least one transaction object in the Event.\n\nThen in order to have a" + " valid GoAML document, please follow these guidelines:\n- For each transaction object, use either a" + " bank-account, person, or legal-entity object to describe the origin of the transaction, and again one of them" + " to describe the target of the transaction.\n- Create an object reference for both origin and target objects" + " of the transaction.\n- A bank-account object needs a signatory, which is a person object, put as object" + " reference of the bank-account.\n- A person can have an address, which is a geolocation object, put as object" + " reference of the person.\n\nSupported relation types for object references that are recommended for each" + " object are the folowing:\n- transaction:\n\t- 'from', 'from_my_client': Origin of the transaction - at least" + " one of them is required.\n\t- 'to', 'to_my_client': Target of the transaction - at least one of them is" + " required.\n\t- 'address': Location of the transaction - optional.\n- bank-account:\n\t- 'signatory':" + " Signatory of a bank-account - the reference from bank-account to a signatory is required, but the" + " relation-type is optional at the moment since this reference will always describe a signatory.\n\t- 'entity':" + " Entity owning the bank account - optional.\n- person:\n\t- 'address': Address of a person - optional." + ), + "references": ["http://goaml.unodc.org/"], + "input": ( + "MISP objects (transaction, bank-account, person, legal-entity, geolocation), with references, describing" + " financial transactions and their origin and target." + ), + "output": ( + "GoAML format file, describing financial transactions, with their origin and target (bank accounts, persons or" + " entities)." + ), } -moduleconfig = ['rentity_id'] -mispattributes = {'input': ['MISPEvent'], 'output': ['xml file']} +moduleconfig = ["rentity_id"] +mispattributes = {"input": ["MISPEvent"], "output": ["xml file"]} outputFileExtension = "xml" responseType = "application/xml" -objects_to_parse = ['transaction', 'bank-account', 'person', 'entity', 'geolocation'] - -goAMLmapping = {'bank-account': {'bank-account': 't_account', 'institution-name': 'institution_name', - 'institution-code': 'institution_code', 'iban': 'iban', 'swift': 'swift', - 'branch': 'branch', 'non-banking-institution': 'non_bank_institution', - 'account': 'account', 'currency-code': 'currency_code', - 'account-name': 'account_name', 'client-number': 'client_number', - 'personal-account-type': 'personal_account_type', 'opened': 'opened', - 'closed': 'closed', 'balance': 'balance', 'status-code': 'status_code', - 'beneficiary': 'beneficiary', 'beneficiary-comment': 'beneficiary_comment', - 'comments': 'comments'}, - 'person': {'person': 't_person', 'text': 'comments', 'first-name': 'first_name', - 'middle-name': 'middle_name', 'last-name': 'last_name', 'title': 'title', - 'mothers-name': 'mothers_name', 'alias': 'alias', 'date-of-birth': 'birthdate', - 'place-of-birth': 'birth_place', 'gender': 'gender', 'nationality': 'nationality1', - 'passport-number': 'passport_number', 'passport-country': 'passport_country', - 'social-security-number': 'ssn', 'identity-card-number': 'id_number'}, - 'geolocation': {'geolocation': 'location', 'city': 'city', 'region': 'state', - 'country': 'country_code', 'address': 'address', 'zipcode': 'zip'}, - 'transaction': {'transaction': 'transaction', 'transaction-number': 'transactionnumber', - 'date': 'date_transaction', 'location': 'transaction_location', - 'transmode-code': 'transmode_code', 'amount': 'amount_local', - 'transmode-comment': 'transmode_comment', 'date-posting': 'date_posting', - 'teller': 'teller', 'authorized': 'authorized', - 'text': 'transaction_description'}, - 'legal-entity': {'legal-entity': 'entity', 'name': 'name', 'business': 'business', - 'commercial-name': 'commercial_name', 'phone-number': 'phone', - 'legal-form': 'incorporation_legal_form', - 'registration-number': 'incorporation_number'}} - -referencesMapping = {'bank-account': {'aml_type': '{}_account', 'bracket': 't_{}'}, - 'person': {'transaction': {'aml_type': '{}_person', 'bracket': 't_{}'}, 'bank-account': {'aml_type': 't_person', 'bracket': 'signatory'}}, - 'legal-entity': {'transaction': {'aml_type': '{}_entity', 'bracket': 't_{}'}, 'bank-account': {'aml_type': 't_entity'}}, - 'geolocation': {'aml_type': 'address', 'bracket': 'addresses'}} +objects_to_parse = ["transaction", "bank-account", "person", "entity", "geolocation"] + +goAMLmapping = { + "bank-account": { + "bank-account": "t_account", + "institution-name": "institution_name", + "institution-code": "institution_code", + "iban": "iban", + "swift": "swift", + "branch": "branch", + "non-banking-institution": "non_bank_institution", + "account": "account", + "currency-code": "currency_code", + "account-name": "account_name", + "client-number": "client_number", + "personal-account-type": "personal_account_type", + "opened": "opened", + "closed": "closed", + "balance": "balance", + "status-code": "status_code", + "beneficiary": "beneficiary", + "beneficiary-comment": "beneficiary_comment", + "comments": "comments", + }, + "person": { + "person": "t_person", + "text": "comments", + "first-name": "first_name", + "middle-name": "middle_name", + "last-name": "last_name", + "title": "title", + "mothers-name": "mothers_name", + "alias": "alias", + "date-of-birth": "birthdate", + "place-of-birth": "birth_place", + "gender": "gender", + "nationality": "nationality1", + "passport-number": "passport_number", + "passport-country": "passport_country", + "social-security-number": "ssn", + "identity-card-number": "id_number", + }, + "geolocation": { + "geolocation": "location", + "city": "city", + "region": "state", + "country": "country_code", + "address": "address", + "zipcode": "zip", + }, + "transaction": { + "transaction": "transaction", + "transaction-number": "transactionnumber", + "date": "date_transaction", + "location": "transaction_location", + "transmode-code": "transmode_code", + "amount": "amount_local", + "transmode-comment": "transmode_comment", + "date-posting": "date_posting", + "teller": "teller", + "authorized": "authorized", + "text": "transaction_description", + }, + "legal-entity": { + "legal-entity": "entity", + "name": "name", + "business": "business", + "commercial-name": "commercial_name", + "phone-number": "phone", + "legal-form": "incorporation_legal_form", + "registration-number": "incorporation_number", + }, +} + +referencesMapping = { + "bank-account": {"aml_type": "{}_account", "bracket": "t_{}"}, + "person": { + "transaction": {"aml_type": "{}_person", "bracket": "t_{}"}, + "bank-account": {"aml_type": "t_person", "bracket": "signatory"}, + }, + "legal-entity": { + "transaction": {"aml_type": "{}_entity", "bracket": "t_{}"}, + "bank-account": {"aml_type": "t_entity"}, + }, + "geolocation": {"aml_type": "address", "bracket": "addresses"}, +} class GoAmlGeneration(object): @@ -75,41 +148,57 @@ def parse_objects(self): for obj in self.misp_event.objects: obj_type = obj.name uuids[obj_type].append(obj.uuid) - if obj_type == 'bank-account': + if obj_type == "bank-account": try: - report_code.append(obj.get_attributes_by_relation('report-code')[0].value.split(' ')[0]) - currency_code.append(obj.get_attributes_by_relation('currency-code')[0].value) + report_code.append(obj.get_attributes_by_relation("report-code")[0].value.split(" ")[0]) + currency_code.append(obj.get_attributes_by_relation("currency-code")[0].value) except IndexError: - print('report_code or currency_code error') - self.uuids, self.report_codes, self.currency_codes = uuids, report_code, currency_code + print("report_code or currency_code error") + self.uuids, self.report_codes, self.currency_codes = ( + uuids, + report_code, + currency_code, + ) def build_xml(self): - self.xml = {'header': "{}E".format(self.config), - 'data': ""} + self.xml = { + "header": "{}E".format(self.config), + "data": "", + } if "STR" in self.report_codes: report_code = "STR" else: report_code = Counter(self.report_codes).most_common(1)[0][0] - self.xml['header'] += "{}".format(report_code) - submission_date = str(self.misp_event.timestamp).replace(' ', 'T') - self.xml['header'] += "{}".format(submission_date) - self.xml['header'] += "{}".format(Counter(self.currency_codes).most_common(1)[0][0]) - for trans_uuid in self.uuids.get('transaction'): - self.itterate('transaction', 'transaction', trans_uuid, 'data') - person_to_parse = [person_uuid for person_uuid in self.uuids.get('person') if person_uuid not in self.parsed_uuids.get('person')] + self.xml["header"] += "{}".format(report_code) + submission_date = str(self.misp_event.timestamp).replace(" ", "T") + self.xml["header"] += "{}".format(submission_date) + self.xml["header"] += "{}".format( + Counter(self.currency_codes).most_common(1)[0][0] + ) + for trans_uuid in self.uuids.get("transaction"): + self.itterate("transaction", "transaction", trans_uuid, "data") + person_to_parse = [ + person_uuid + for person_uuid in self.uuids.get("person") + if person_uuid not in self.parsed_uuids.get("person") + ] if len(person_to_parse) == 1: - self.itterate('person', 'reporting_person', person_to_parse[0], 'header') + self.itterate("person", "reporting_person", person_to_parse[0], "header") try: - location_to_parse = [location_uuid for location_uuid in self.uuids.get('geolocation') if location_uuid not in self.parsed_uuids.get('geolocation')] + location_to_parse = [ + location_uuid + for location_uuid in self.uuids.get("geolocation") + if location_uuid not in self.parsed_uuids.get("geolocation") + ] if len(location_to_parse) == 1: - self.itterate('geolocation', 'location', location_to_parse[0], 'header') + self.itterate("geolocation", "location", location_to_parse[0], "header") except TypeError: pass - self.xml['data'] += "" + self.xml["data"] += "" def itterate(self, object_type, aml_type, uuid, xml_part): obj = self.misp_event.get_object_by_uuid(uuid) - if object_type == 'transaction': + if object_type == "transaction": self.xml[xml_part] += "<{}>".format(aml_type) self.fill_xml_transaction(object_type, obj.attributes, xml_part) self.parsed_uuids[object_type].append(uuid) @@ -117,11 +206,16 @@ def itterate(self, object_type, aml_type, uuid, xml_part): self.parseObjectReferences(object_type, xml_part, obj.ObjectReference) self.xml[xml_part] += "".format(aml_type) else: - if 'to_' in aml_type or 'from_' in aml_type: - relation_type = aml_type.split('_')[0] - self.xml[xml_part] += "<{0}_funds_code>{1}".format(relation_type, self.from_and_to_fields[relation_type]['funds'].split(' ')[0]) + if "to_" in aml_type or "from_" in aml_type: + relation_type = aml_type.split("_")[0] + self.xml[xml_part] += "<{0}_funds_code>{1}".format( + relation_type, + self.from_and_to_fields[relation_type]["funds"].split(" ")[0], + ) self.itterate_normal_case(object_type, obj, aml_type, uuid, xml_part) - self.xml[xml_part] += "<{0}_country>{1}".format(relation_type, self.from_and_to_fields[relation_type]['country']) + self.xml[xml_part] += "<{0}_country>{1}".format( + relation_type, self.from_and_to_fields[relation_type]["country"] + ) else: self.itterate_normal_case(object_type, obj, aml_type, uuid, xml_part) @@ -136,63 +230,74 @@ def itterate_normal_case(self, object_type, obj, aml_type, uuid, xml_part): def parseObjectReferences(self, object_type, xml_part, references): for ref in references: next_uuid = ref.referenced_uuid - next_object_type = ref.Object.get('name') + next_object_type = ref.Object.get("name") relationship_type = ref.relationship_type self.parse_references(object_type, next_object_type, next_uuid, relationship_type, xml_part) def fill_xml_transaction(self, object_type, attributes, xml_part): - from_and_to_fields = {'from': {}, 'to': {}} + from_and_to_fields = {"from": {}, "to": {}} for attribute in attributes: object_relation = attribute.object_relation attribute_value = attribute.value - if object_relation == 'date-posting': + if object_relation == "date-posting": self.xml[xml_part] += "True" - elif object_relation in ('from-funds-code', 'to-funds-code'): - relation_type, field, _ = object_relation.split('-') + elif object_relation in ("from-funds-code", "to-funds-code"): + relation_type, field, _ = object_relation.split("-") from_and_to_fields[relation_type][field] = attribute_value continue - elif object_relation in ('from-country', 'to-country'): - relation_type, field = object_relation.split('-') + elif object_relation in ("from-country", "to-country"): + relation_type, field = object_relation.split("-") from_and_to_fields[relation_type][field] = attribute_value continue try: - self.xml[xml_part] += "<{0}>{1}".format(goAMLmapping[object_type][object_relation], attribute_value) + self.xml[xml_part] += "<{0}>{1}".format( + goAMLmapping[object_type][object_relation], attribute_value + ) except KeyError: pass self.from_and_to_fields = from_and_to_fields def fill_xml(self, object_type, obj, xml_part): - if obj.name == 'bank-account': + if obj.name == "bank-account": for attribute in obj.attributes: - if attribute.object_relation in ('personal-account-type', 'status-code'): - attribute_value = attribute.value.split(' - ')[0] + if attribute.object_relation in ( + "personal-account-type", + "status-code", + ): + attribute_value = attribute.value.split(" - ")[0] else: attribute_value = attribute.value try: - self.xml[xml_part] += "<{0}>{1}".format(goAMLmapping[object_type][attribute.object_relation], attribute_value) + self.xml[xml_part] += "<{0}>{1}".format( + goAMLmapping[object_type][attribute.object_relation], + attribute_value, + ) except KeyError: pass else: for attribute in obj.attributes: try: - self.xml[xml_part] += "<{0}>{1}".format(goAMLmapping[object_type][attribute.object_relation], attribute.value) + self.xml[xml_part] += "<{0}>{1}".format( + goAMLmapping[object_type][attribute.object_relation], + attribute.value, + ) except KeyError: pass def parse_references(self, object_type, next_object_type, uuid, relationship_type, xml_part): reference = referencesMapping[next_object_type] try: - next_aml_type = reference[object_type].get('aml_type').format(relationship_type.split('_')[0]) + next_aml_type = reference[object_type].get("aml_type").format(relationship_type.split("_")[0]) try: - bracket = reference[object_type].get('bracket').format(relationship_type) + bracket = reference[object_type].get("bracket").format(relationship_type) self.xml[xml_part] += "<{}>".format(bracket) self.itterate(next_object_type, next_aml_type, uuid, xml_part) self.xml[xml_part] += "".format(bracket) except KeyError: self.itterate(next_object_type, next_aml_type, uuid, xml_part) except KeyError: - next_aml_type = reference.get('aml_type').format(relationship_type.split('_')[0]) - bracket = reference.get('bracket').format(relationship_type) + next_aml_type = reference.get("aml_type").format(relationship_type.split("_")[0]) + bracket = reference.get("bracket").format(relationship_type) self.xml[xml_part] += "<{}>".format(bracket) self.itterate(next_object_type, next_aml_type, uuid, xml_part) self.xml[xml_part] += "".format(bracket) @@ -202,54 +307,57 @@ def handler(q=False): if q is False: return False request = json.loads(q) - if 'data' not in request: + if "data" not in request: return False - if not request.get('config') and not request['config'].get('rentity_id'): - misperrors['error'] = "Configuration error." + if not request.get("config") and not request["config"].get("rentity_id"): + misperrors["error"] = "Configuration error." return misperrors - config = request['config'].get('rentity_id') + config = request["config"].get("rentity_id") export_doc = GoAmlGeneration(config) - export_doc.from_event(request['data'][0]) + export_doc.from_event(request["data"][0]) if not export_doc.misp_event.Object: - misperrors['error'] = "There is no object in this event." + misperrors["error"] = "There is no object in this event." return misperrors types = [] for obj in export_doc.misp_event.Object: types.append(obj.name) - if 'transaction' not in types: - misperrors['error'] = "There is no transaction object in this event." + if "transaction" not in types: + misperrors["error"] = "There is no transaction object in this event." return misperrors export_doc.parse_objects() export_doc.build_xml() - exp_doc = "{}{}".format(export_doc.xml.get('header'), export_doc.xml.get('data')) - return {'response': [], 'data': str(base64.b64encode(bytes(exp_doc, 'utf-8')), 'utf-8')} + exp_doc = "{}{}".format(export_doc.xml.get("header"), export_doc.xml.get("data")) + return { + "response": [], + "data": str(base64.b64encode(bytes(exp_doc, "utf-8")), "utf-8"), + } def introspection(): modulesetup = {} try: responseType - modulesetup['responseType'] = responseType + modulesetup["responseType"] = responseType except NameError: pass try: userConfig - modulesetup['userConfig'] = userConfig + modulesetup["userConfig"] = userConfig except NameError: pass try: outputFileExtension - modulesetup['outputFileExtension'] = outputFileExtension + modulesetup["outputFileExtension"] = outputFileExtension except NameError: pass try: inputSource - moduleSetup['inputSource'] = inputSource + moduleSetup["inputSource"] = inputSource except NameError: pass return modulesetup def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/export_mod/liteexport.py b/misp_modules/modules/export_mod/liteexport.py index d980c9f9e..5462b4aea 100755 --- a/misp_modules/modules/export_mod/liteexport.py +++ b/misp_modules/modules/export_mod/liteexport.py @@ -1,20 +1,24 @@ -import json import base64 +import json -misperrors = {'error': 'Error'} +misperrors = {"error": "Error"} moduleinfo = { - 'version': '1', - 'author': 'TM', - 'description': 'Lite export of a MISP event.', - 'module-type': ['export'], - 'name': 'Lite Export', - 'logo': '', - 'requirements': [], - 'features': 'This module is simply producing a json MISP event format file, but exporting only Attributes from the Event. Thus, MISP Events exported with this module should have attributes that are not internal references, otherwise the resulting event would be empty.', - 'references': [], - 'input': 'MISP Event attributes', - 'output': 'Lite MISP Event', + "version": "1", + "author": "TM", + "description": "Lite export of a MISP event.", + "module-type": ["export"], + "name": "Lite Export", + "logo": "", + "requirements": [], + "features": ( + "This module is simply producing a json MISP event format file, but exporting only Attributes from the Event." + " Thus, MISP Events exported with this module should have attributes that are not internal references," + " otherwise the resulting event would be empty." + ), + "references": [], + "input": "MISP Event attributes", + "output": "Lite MISP Event", } moduleconfig = ["indent_json_export"] @@ -36,62 +40,66 @@ def handler(q=False): else: config = {"indent_json_export": None} - if config['indent_json_export'] is not None: + if config["indent_json_export"] is not None: try: - config['indent_json_export'] = int(config['indent_json_export']) + config["indent_json_export"] = int(config["indent_json_export"]) except Exception: - config['indent_json_export'] = None + config["indent_json_export"] = None - if 'data' not in request: + if "data" not in request: return False # ~ Misp json structur - liteEvent = {'Event': {}} + liteEvent = {"Event": {}} - for evt in request['data']: - rawEvent = evt['Event'] - liteEvent['Event']['info'] = rawEvent['info'] - liteEvent['Event']['Attribute'] = [] + for evt in request["data"]: + rawEvent = evt["Event"] + liteEvent["Event"]["info"] = rawEvent["info"] + liteEvent["Event"]["Attribute"] = [] - attrs = evt['Attribute'] + attrs = evt["Attribute"] for attr in attrs: - if 'Internal reference' not in attr['category']: + if "Internal reference" not in attr["category"]: liteAttr = {} - liteAttr['category'] = attr['category'] - liteAttr['type'] = attr['type'] - liteAttr['value'] = attr['value'] - liteEvent['Event']['Attribute'].append(liteAttr) + liteAttr["category"] = attr["category"] + liteAttr["type"] = attr["type"] + liteAttr["value"] = attr["value"] + liteEvent["Event"]["Attribute"].append(liteAttr) - return {'response': [], - 'data': str(base64.b64encode(bytes( - json.dumps(liteEvent, indent=config['indent_json_export']), 'utf-8')), 'utf-8')} + return { + "response": [], + "data": str( + base64.b64encode(bytes(json.dumps(liteEvent, indent=config["indent_json_export"]), "utf-8")), + "utf-8", + ), + } def introspection(): modulesetup = {} try: responseType - modulesetup['responseType'] = responseType + modulesetup["responseType"] = responseType except NameError: pass try: userConfig - modulesetup['userConfig'] = userConfig + modulesetup["userConfig"] = userConfig except NameError: pass try: outputFileExtension - modulesetup['outputFileExtension'] = outputFileExtension + modulesetup["outputFileExtension"] = outputFileExtension except NameError: pass try: inputSource - modulesetup['inputSource'] = inputSource + modulesetup["inputSource"] = inputSource except NameError: pass return modulesetup def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/export_mod/mass_eql_export.py b/misp_modules/modules/export_mod/mass_eql_export.py index d6ed1ea39..4c3b79df2 100644 --- a/misp_modules/modules/export_mod/mass_eql_export.py +++ b/misp_modules/modules/export_mod/mass_eql_export.py @@ -1,6 +1,7 @@ """ Export module for converting MISP events into Endgame EQL queries """ + import base64 import io import json @@ -9,37 +10,35 @@ misperrors = {"error": "Error"} moduleinfo = { - 'version': '0.1', - 'author': '92 COS DOM', - 'description': 'Export MISP event in Event Query Language', - 'module-type': ['export'], - 'name': 'EQL Query Export', - 'logo': 'eql.png', - 'requirements': [], - 'features': 'This module produces EQL queries for all relevant attributes in a MISP event.', - 'references': ['https://eql.readthedocs.io/en/latest/'], - 'input': 'MISP Event attributes', - 'output': 'Text file containing one or more EQL queries', + "version": "0.1", + "author": "92 COS DOM", + "description": "Export MISP event in Event Query Language", + "module-type": ["export"], + "name": "EQL Query Export", + "logo": "eql.png", + "requirements": [], + "features": "This module produces EQL queries for all relevant attributes in a MISP event.", + "references": ["https://eql.readthedocs.io/en/latest/"], + "input": "MISP Event attributes", + "output": "Text file containing one or more EQL queries", } # Map of MISP fields => Endgame fields fieldmap = { "ip-src": "source_address", "ip-dst": "destination_address", - "filename": "file_name" + "filename": "file_name", } # Describe what events have what fields event_types = { "source_address": "network", "destination_address": "network", - "file_name": "file" + "file_name": "file", } # combine all the MISP fields from fieldmap into one big list -mispattributes = { - "input": list(fieldmap.keys()) -} +mispattributes = {"input": list(fieldmap.keys())} def handler(q=False): @@ -74,10 +73,13 @@ def handler(q=False): for value in queryDict[query].keys(): if i != 0: response.write(" or\n") - response.write("\t{} == \"{}\"".format(queryDict[query][value], value)) + response.write('\t{} == "{}"'.format(queryDict[query][value], value)) i += 1 - return {"response": [], "data": str(base64.b64encode(bytes(response.getvalue(), 'utf-8')), 'utf-8')} + return { + "response": [], + "data": str(base64.b64encode(bytes(response.getvalue(), "utf-8")), "utf-8"), + } def introspection(): @@ -91,7 +93,7 @@ def introspection(): "responseType": "application/txt", "outputFileExtension": "txt", "userConfig": {}, - "inputSource": [] + "inputSource": [], } return modulesetup diff --git a/misp_modules/modules/export_mod/nexthinkexport.py b/misp_modules/modules/export_mod/nexthinkexport.py index f0c7f3e71..d5722924d 100755 --- a/misp_modules/modules/export_mod/nexthinkexport.py +++ b/misp_modules/modules/export_mod/nexthinkexport.py @@ -9,95 +9,108 @@ misperrors = {"error": "Error"} -types_to_use = ['sha1', 'sha256', 'md5', 'domain'] +types_to_use = ["sha1", "sha256", "md5", "domain"] -userConfig = { - -} +userConfig = {} moduleconfig = ["Period"] -inputSource = ['event'] +inputSource = ["event"] -outputFileExtension = 'nxql' -responseType = 'application/txt' +outputFileExtension = "nxql" +responseType = "application/txt" moduleinfo = { - 'version': '1.0', - 'author': 'Julien Bachmann, Hacknowledge', - 'description': 'Nexthink NXQL query export module', - 'module-type': ['export'], - 'name': 'Nexthink NXQL Export', - 'logo': 'nexthink.svg', - 'requirements': [], - 'features': 'This module export an event as Nexthink NXQL queries that can then be used in your own python3 tool or from wget/powershell', - 'references': ['https://doc.nexthink.com/Documentation/Nexthink/latest/APIAndIntegrations/IntroducingtheWebAPIV2'], - 'input': 'MISP Event attributes', - 'output': 'Nexthink NXQL queries', + "version": "1.0", + "author": "Julien Bachmann, Hacknowledge", + "description": "Nexthink NXQL query export module", + "module-type": ["export"], + "name": "Nexthink NXQL Export", + "logo": "nexthink.svg", + "requirements": [], + "features": ( + "This module export an event as Nexthink NXQL queries that can then be used in your own python3 tool or from" + " wget/powershell" + ), + "references": ["https://doc.nexthink.com/Documentation/Nexthink/latest/APIAndIntegrations/IntroducingtheWebAPIV2"], + "input": "MISP Event attributes", + "output": "Nexthink NXQL queries", } def handle_sha1(value, period): - query = '''select ((binary (executable_name version)) (user (name)) (device (name last_ip_address)) (execution (binary_path start_time))) + query = """select ((binary (executable_name version)) (user (name)) (device (name last_ip_address)) (execution (binary_path start_time))) (from (binary user device execution) (where binary (eq sha1 (sha1 %s))) (between now-%s now)) (limit 1000) - ''' % (value, period) - return query.replace('\n', ' ') + """ % ( + value, + period, + ) + return query.replace("\n", " ") def handle_sha256(value, period): - query = '''select ((binary (executable_name version)) (user (name)) (device (name last_ip_address)) (execution (binary_path start_time))) + query = """select ((binary (executable_name version)) (user (name)) (device (name last_ip_address)) (execution (binary_path start_time))) (from (binary user device execution) (where binary (eq sha256 (sha256 %s))) (between now-%s now)) (limit 1000) - ''' % (value, period) - return query.replace('\n', ' ') + """ % ( + value, + period, + ) + return query.replace("\n", " ") def handle_md5(value, period): - query = '''select ((binary (executable_name version)) (user (name)) (device (name last_ip_address)) (execution (binary_path start_time))) + query = """select ((binary (executable_name version)) (user (name)) (device (name last_ip_address)) (execution (binary_path start_time))) (from (binary user device execution) (where binary (eq hash (md5 %s))) (between now-%s now)) (limit 1000) - ''' % (value, period) - return query.replace('\n', ' ') + """ % ( + value, + period, + ) + return query.replace("\n", " ") def handle_domain(value, period): - query = '''select ((device name) (device (name last_ip_address)) (user name)(user department) (binary executable_name)(binary application_name)(binary description)(binary application_category)(binary (executable_name version)) (binary #"Suspicious binary")(binary first_seen)(binary last_seen)(binary threat_level)(binary hash) (binary paths) + query = """select ((device name) (device (name last_ip_address)) (user name)(user department) (binary executable_name)(binary application_name)(binary description)(binary application_category)(binary (executable_name version)) (binary #"Suspicious binary")(binary first_seen)(binary last_seen)(binary threat_level)(binary hash) (binary paths) (destination name)(domain name) (domain domain_category)(domain hosting_country)(domain protocol)(domain threat_level) (port port_number)(web_request incoming_traffic)(web_request outgoing_traffic)) (from (web_request device user binary executable destination domain port) (where domain (eq name(string %s))) (between now-%s now)) (limit 1000) - ''' % (value, period) - return query.replace('\n', ' ') + """ % ( + value, + period, + ) + return query.replace("\n", " ") handlers = { - 'sha1': handle_sha1, - 'sha256': handle_sha256, - 'md5': handle_md5, - 'domain': handle_domain + "sha1": handle_sha1, + "sha256": handle_sha256, + "md5": handle_md5, + "domain": handle_domain, } def handler(q=False): if q is False: return False - r = {'results': []} + r = {"results": []} request = json.loads(q) config = request.get("config", {"Period": ""}) - output = '' + output = "" for event in request["data"]: for attribute in event["Attribute"]: - if attribute['type'] in types_to_use: - output = output + handlers[attribute['type']](attribute['value'], config['Period']) + '\n' - r = {"response": [], "data": str(base64.b64encode(bytes(output, 'utf-8')), 'utf-8')} + if attribute["type"] in types_to_use: + output = output + handlers[attribute["type"]](attribute["value"], config["Period"]) + "\n" + r = {"response": [], "data": str(base64.b64encode(bytes(output, "utf-8")), "utf-8")} return r @@ -105,27 +118,27 @@ def introspection(): modulesetup = {} try: responseType - modulesetup['responseType'] = responseType + modulesetup["responseType"] = responseType except NameError: pass try: userConfig - modulesetup['userConfig'] = userConfig + modulesetup["userConfig"] = userConfig except NameError: pass try: outputFileExtension - modulesetup['outputFileExtension'] = outputFileExtension + modulesetup["outputFileExtension"] = outputFileExtension except NameError: pass try: inputSource - modulesetup['inputSource'] = inputSource + modulesetup["inputSource"] = inputSource except NameError: pass return modulesetup def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/export_mod/osqueryexport.py b/misp_modules/modules/export_mod/osqueryexport.py index 3022a563d..81aa7d2f5 100755 --- a/misp_modules/modules/export_mod/osqueryexport.py +++ b/misp_modules/modules/export_mod/osqueryexport.py @@ -9,89 +9,103 @@ misperrors = {"error": "Error"} -types_to_use = ['regkey', 'regkey|value', 'mutex', 'windows-service-displayname', 'windows-scheduled-task', 'yara'] +types_to_use = [ + "regkey", + "regkey|value", + "mutex", + "windows-service-displayname", + "windows-scheduled-task", + "yara", +] -userConfig = { - -} +userConfig = {} moduleconfig = [] -inputSource = ['event'] +inputSource = ["event"] -outputFileExtension = 'conf' -responseType = 'application/txt' +outputFileExtension = "conf" +responseType = "application/txt" moduleinfo = { - 'version': '1.0', - 'author': 'Julien Bachmann, Hacknowledge', - 'description': 'OSQuery export of a MISP event.', - 'module-type': ['export'], - 'name': 'OSQuery Export', - 'logo': 'osquery.png', - 'requirements': [], - 'features': 'This module export an event as osquery queries that can be used in packs or in fleet management solution like Kolide.', - 'references': [], - 'input': 'MISP Event attributes', - 'output': 'osquery SQL queries', + "version": "1.0", + "author": "Julien Bachmann, Hacknowledge", + "description": "OSQuery export of a MISP event.", + "module-type": ["export"], + "name": "OSQuery Export", + "logo": "osquery.png", + "requirements": [], + "features": ( + "This module export an event as osquery queries that can be used in packs or in fleet management solution like" + " Kolide." + ), + "references": [], + "input": "MISP Event attributes", + "output": "osquery SQL queries", } def handle_regkey(value): - rep = {'HKCU': 'HKEY_USERS\\%', 'HKLM': 'HKEY_LOCAL_MACHINE'} + rep = {"HKCU": "HKEY_USERS\\%", "HKLM": "HKEY_LOCAL_MACHINE"} rep = dict((re.escape(k), v) for k, v in rep.items()) pattern = re.compile("|".join(rep.keys())) value = pattern.sub(lambda m: rep[re.escape(m.group(0))], value) - return 'SELECT * FROM registry WHERE path LIKE \'%s\';' % value + return "SELECT * FROM registry WHERE path LIKE '%s';" % value def handle_regkeyvalue(value): - key, value = value.split('|') - rep = {'HKCU': 'HKEY_USERS\\%', 'HKLM': 'HKEY_LOCAL_MACHINE'} + key, value = value.split("|") + rep = {"HKCU": "HKEY_USERS\\%", "HKLM": "HKEY_LOCAL_MACHINE"} rep = dict((re.escape(k), v) for k, v in rep.items()) pattern = re.compile("|".join(rep.keys())) key = pattern.sub(lambda m: rep[re.escape(m.group(0))], key) - return 'SELECT * FROM registry WHERE path LIKE \'%s\' AND data LIKE \'%s\';' % (key, value) + return "SELECT * FROM registry WHERE path LIKE '%s' AND data LIKE '%s';" % ( + key, + value, + ) def handle_mutex(value): - return 'SELECT * FROM winbaseobj WHERE object_name LIKE \'%s\';' % value + return "SELECT * FROM winbaseobj WHERE object_name LIKE '%s';" % value def handle_service(value): - return 'SELECT * FROM services WHERE display_name LIKE \'%s\' OR name like \'%s\';' % (value, value) + return "SELECT * FROM services WHERE display_name LIKE '%s' OR name like '%s';" % ( + value, + value, + ) def handle_yara(value): - return 'not implemented yet, not sure it\'s easily feasible w/o dropping the sig on the hosts first' + return "not implemented yet, not sure it's easily feasible w/o dropping the sig on the hosts first" def handle_scheduledtask(value): - return 'SELECT * FROM scheduled_tasks WHERE name LIKE \'%s\';' % value + return "SELECT * FROM scheduled_tasks WHERE name LIKE '%s';" % value handlers = { - 'regkey': handle_regkey, - 'regkey|value': handle_regkeyvalue, - 'mutex': handle_mutex, - 'windows-service-displayname': handle_service, - 'windows-scheduled-task': handle_scheduledtask, - 'yara': handle_yara + "regkey": handle_regkey, + "regkey|value": handle_regkeyvalue, + "mutex": handle_mutex, + "windows-service-displayname": handle_service, + "windows-scheduled-task": handle_scheduledtask, + "yara": handle_yara, } def handler(q=False): if q is False: return False - r = {'results': []} + r = {"results": []} request = json.loads(q) - output = '' + output = "" for event in request["data"]: for attribute in event["Attribute"]: - if attribute['type'] in types_to_use: - output = output + handlers[attribute['type']](attribute['value']) + '\n' - r = {"response": [], "data": str(base64.b64encode(bytes(output, 'utf-8')), 'utf-8')} + if attribute["type"] in types_to_use: + output = output + handlers[attribute["type"]](attribute["value"]) + "\n" + r = {"response": [], "data": str(base64.b64encode(bytes(output, "utf-8")), "utf-8")} return r @@ -99,27 +113,27 @@ def introspection(): modulesetup = {} try: responseType - modulesetup['responseType'] = responseType + modulesetup["responseType"] = responseType except NameError: pass try: userConfig - modulesetup['userConfig'] = userConfig + modulesetup["userConfig"] = userConfig except NameError: pass try: outputFileExtension - modulesetup['outputFileExtension'] = outputFileExtension + modulesetup["outputFileExtension"] = outputFileExtension except NameError: pass try: inputSource - modulesetup['inputSource'] = inputSource + modulesetup["inputSource"] = inputSource except NameError: pass return modulesetup def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/export_mod/pdfexport.py b/misp_modules/modules/export_mod/pdfexport.py index 53ea17d72..c561fb33c 100755 --- a/misp_modules/modules/export_mod/pdfexport.py +++ b/misp_modules/modules/export_mod/pdfexport.py @@ -6,41 +6,66 @@ from pymisp import MISPEvent from pymisp.tools import reportlab_generator -misperrors = {'error': 'Error'} +misperrors = {"error": "Error"} moduleinfo = { - 'version': '2', - 'author': 'Vincent Falconieri (prev. Raphaël Vinot)', - 'description': 'Simple export of a MISP event to PDF.', - 'module-type': ['export'], - 'name': 'Event to PDF Export', - 'require_standard_format': True, - 'logo': '', - 'requirements': ['PyMISP', 'reportlab'], - 'features': "The module takes care of the PDF file building, and work with any MISP Event. Except the requirement of reportlab, used to create the file, there is no special feature concerning the Event. Some parameters can be given through the config dict. 'MISP_base_url_for_dynamic_link' is your MISP URL, to attach an hyperlink to your event on your MISP instance from the PDF. Keep it clear to avoid hyperlinks in the generated pdf.\n 'MISP_name_for_metadata' is your CERT or MISP instance name. Used as text in the PDF' metadata\n 'Activate_textual_description' is a boolean (True or void) to activate the textual description/header abstract of an event\n 'Activate_galaxy_description' is a boolean (True or void) to activate the description of event related galaxies.\n 'Activate_related_events' is a boolean (True or void) to activate the description of related event. Be aware this might leak information on confidential events linked to the current event !\n 'Activate_internationalization_fonts' is a boolean (True or void) to activate Noto fonts instead of default fonts (Helvetica). This allows the support of CJK alphabet. Be sure to have followed the procedure to download Noto fonts (~70Mo) in the right place (/tools/pdf_fonts/Noto_TTF), to allow PyMisp to find and use them during PDF generation.\n 'Custom_fonts_path' is a text (path or void) to the TTF file of your choice, to create the PDF with it. Be aware the PDF won't support bold/italic/special style anymore with this option ", - 'references': ['https://acrobat.adobe.com/us/en/acrobat/about-adobe-pdf.html'], - 'input': 'MISP Event', - 'output': 'MISP Event in a PDF file.', + "version": "2", + "author": "Vincent Falconieri (prev. Raphaël Vinot)", + "description": "Simple export of a MISP event to PDF.", + "module-type": ["export"], + "name": "Event to PDF Export", + "require_standard_format": True, + "logo": "", + "requirements": ["PyMISP", "reportlab"], + "features": ( + "The module takes care of the PDF file building, and work with any MISP Event. Except the requirement of" + " reportlab, used to create the file, there is no special feature concerning the Event. Some parameters can be" + " given through the config dict. 'MISP_base_url_for_dynamic_link' is your MISP URL, to attach an hyperlink to" + " your event on your MISP instance from the PDF. Keep it clear to avoid hyperlinks in the generated pdf.\n " + " 'MISP_name_for_metadata' is your CERT or MISP instance name. Used as text in the PDF' metadata\n " + " 'Activate_textual_description' is a boolean (True or void) to activate the textual description/header" + " abstract of an event\n 'Activate_galaxy_description' is a boolean (True or void) to activate the description" + " of event related galaxies.\n 'Activate_related_events' is a boolean (True or void) to activate the" + " description of related event. Be aware this might leak information on confidential events linked to the" + " current event !\n 'Activate_internationalization_fonts' is a boolean (True or void) to activate Noto fonts" + " instead of default fonts (Helvetica). This allows the support of CJK alphabet. Be sure to have followed the" + " procedure to download Noto fonts (~70Mo) in the right place (/tools/pdf_fonts/Noto_TTF), to allow PyMisp to" + " find and use them during PDF generation.\n 'Custom_fonts_path' is a text (path or void) to the TTF file of" + " your choice, to create the PDF with it. Be aware the PDF won't support bold/italic/special style anymore with" + " this option " + ), + "references": ["https://acrobat.adobe.com/us/en/acrobat/about-adobe-pdf.html"], + "input": "MISP Event", + "output": "MISP Event in a PDF file.", } # config fields that your code expects from the site admin -moduleconfig = ["MISP_base_url_for_dynamic_link", "MISP_name_for_metadata", "Activate_textual_description", "Activate_galaxy_description", "Activate_related_events", "Activate_internationalization_fonts", "Custom_fonts_path"] +moduleconfig = [ + "MISP_base_url_for_dynamic_link", + "MISP_name_for_metadata", + "Activate_textual_description", + "Activate_galaxy_description", + "Activate_related_events", + "Activate_internationalization_fonts", + "Custom_fonts_path", +] mispattributes = {} outputFileExtension = "pdf" responseType = "application/pdf" -types_to_attach = ['ip-dst', 'url', 'domain'] -objects_to_attach = ['domain-ip'] +types_to_attach = ["ip-dst", "url", "domain"] +objects_to_attach = ["domain-ip"] -class ReportGenerator(): +class ReportGenerator: def __init__(self): - self.report = '' + self.report = "" def from_remote(self, event_id): + from keys import misp_key, misp_url, misp_verifycert from pymisp import PyMISP - from keys import misp_url, misp_key, misp_verifycert + misp = PyMISP(misp_url, misp_key, misp_verifycert) result = misp.get(event_id) self.misp_event = MISPEvent() @@ -57,51 +82,53 @@ def handler(q=False): request = json.loads(q) - if 'data' not in request: + if "data" not in request: return False config = {} # Construct config object for reportlab_generator for config_item in moduleconfig: - if (request.get('config')) and (request['config'].get(config_item) is not None): - config[config_item] = request['config'].get(config_item) + if (request.get("config")) and (request["config"].get(config_item) is not None): + config[config_item] = request["config"].get(config_item) - for evt in request['data']: + for evt in request["data"]: misp_event = MISPEvent() misp_event.load(evt) - pdf = reportlab_generator.get_base64_from_value(reportlab_generator.convert_event_in_pdf_buffer(misp_event, config)) + pdf = reportlab_generator.get_base64_from_value( + reportlab_generator.convert_event_in_pdf_buffer(misp_event, config) + ) - return {'response': [], 'data': str(pdf, 'utf-8')} + return {"response": [], "data": str(pdf, "utf-8")} def introspection(): modulesetup = {} try: responseType - modulesetup['responseType'] = responseType + modulesetup["responseType"] = responseType except NameError: pass try: userConfig - modulesetup['userConfig'] = userConfig + modulesetup["userConfig"] = userConfig except NameError: pass try: outputFileExtension - modulesetup['outputFileExtension'] = outputFileExtension + modulesetup["outputFileExtension"] = outputFileExtension except NameError: pass try: inputSource - modulesetup['inputSource'] = inputSource + modulesetup["inputSource"] = inputSource except NameError: pass return modulesetup def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/export_mod/testexport.py b/misp_modules/modules/export_mod/testexport.py index e1fb6ff5e..b64b1e100 100755 --- a/misp_modules/modules/export_mod/testexport.py +++ b/misp_modules/modules/export_mod/testexport.py @@ -1,45 +1,43 @@ -import json import base64 +import json -misperrors = {'error': 'Error'} - +misperrors = {"error": "Error"} -userConfig = { -} +userConfig = {} moduleconfig = [] # fixed for now, options in the future: # event, attribute, event-collection, attribute-collection -inputSource = ['event'] +inputSource = ["event"] -outputFileExtension = 'txt' -responseType = 'application/txt' +outputFileExtension = "txt" +responseType = "application/txt" moduleinfo = { - 'version': '0.1', - 'author': 'Andras Iklody', - 'description': 'Skeleton export module.', - 'name': 'Test Export', - 'module-type': ['export'], - 'logo': '', - 'requirements': [], - 'features': '', - 'references': [], - 'input': '', - 'output': '', + "version": "0.1", + "author": "Andras Iklody", + "description": "Skeleton export module.", + "name": "Test Export", + "module-type": ["export"], + "logo": "", + "requirements": [], + "features": "", + "references": [], + "input": "", + "output": "", } def handler(q=False): if q is False: return False - r = {'results': []} + r = {"results": []} result = json.loads(q) # noqa - output = '' # Insert your magic here! - r = {"data": base64.b64encode(output.encode('utf-8')).decode('utf-8')} + output = "" # Insert your magic here! + r = {"data": base64.b64encode(output.encode("utf-8")).decode("utf-8")} return r @@ -47,27 +45,27 @@ def introspection(): modulesetup = {} try: responseType - modulesetup['responseType'] = responseType + modulesetup["responseType"] = responseType except NameError: pass try: userConfig - modulesetup['userConfig'] = userConfig + modulesetup["userConfig"] = userConfig except NameError: pass try: outputFileExtension - modulesetup['outputFileExtension'] = outputFileExtension + modulesetup["outputFileExtension"] = outputFileExtension except NameError: pass try: inputSource - modulesetup['inputSource'] = inputSource + modulesetup["inputSource"] = inputSource except NameError: pass return modulesetup def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/export_mod/threatStream_misp_export.py b/misp_modules/modules/export_mod/threatStream_misp_export.py index 68c710f3e..0ebc1be2d 100755 --- a/misp_modules/modules/export_mod/threatStream_misp_export.py +++ b/misp_modules/modules/export_mod/threatStream_misp_export.py @@ -9,21 +9,26 @@ import json import logging - misperrors = {"error": "Error"} moduleinfo = { - 'version': '1.0', - 'author': 'Robert Nixon, based off of the ThreatConnect MISP Module written by the CenturyLink CIRT', - 'description': 'Module to export a structured CSV file for uploading to threatStream.', - 'module-type': ['export'], - 'name': 'ThreatStream Export', - 'logo': 'threatstream.png', - 'requirements': ['csv'], - 'features': 'The module takes a MISP event in input, to look every attribute. Each attribute matching with some predefined types is then exported in a CSV format recognized by ThreatStream.', - 'references': ['https://www.anomali.com/platform/threatstream', 'https://github.com/threatstream'], - 'input': 'MISP Event attributes', - 'output': 'ThreatStream CSV format file', + "version": "1.0", + "author": "Robert Nixon, based off of the ThreatConnect MISP Module written by the CenturyLink CIRT", + "description": "Module to export a structured CSV file for uploading to threatStream.", + "module-type": ["export"], + "name": "ThreatStream Export", + "logo": "threatstream.png", + "requirements": ["csv"], + "features": ( + "The module takes a MISP event in input, to look every attribute. Each attribute matching with some predefined" + " types is then exported in a CSV format recognized by ThreatStream." + ), + "references": [ + "https://www.anomali.com/platform/threatstream", + "https://github.com/threatstream", + ], + "input": "MISP Event attributes", + "output": "ThreatStream CSV format file", } @@ -42,9 +47,7 @@ } # combine all the MISP fields from fieldmap into one big list -mispattributes = { - "input": list(fieldmap.keys()) -} +mispattributes = {"input": list(fieldmap.keys())} def handler(q=False): @@ -66,25 +69,35 @@ def handler(q=False): for event in request["data"]: for attribute in event["Attribute"]: if attribute["type"] in mispattributes["input"]: - logging.debug("Adding %s to structured CSV export of ThreatStream Export", attribute["value"]) + logging.debug( + "Adding %s to structured CSV export of ThreatStream Export", + attribute["value"], + ) if "|" in attribute["type"]: # if the attribute type has multiple values, line it up with the corresponding ThreatStream values in fieldmap indicators = tuple(attribute["value"].split("|")) ts_types = tuple(fieldmap[attribute["type"]].split("|")) for i, indicator in enumerate(indicators): - writer.writerow({ - "value": indicator, - "itype": ts_types[i], - "tags": attribute["comment"] - }) + writer.writerow( + { + "value": indicator, + "itype": ts_types[i], + "tags": attribute["comment"], + } + ) else: - writer.writerow({ - "itype": fieldmap[attribute["type"]], - "value": attribute["value"], - "tags": attribute["comment"] - }) - - return {"response": [], "data": str(base64.b64encode(bytes(response.getvalue(), 'utf-8')), 'utf-8')} + writer.writerow( + { + "itype": fieldmap[attribute["type"]], + "value": attribute["value"], + "tags": attribute["comment"], + } + ) + + return { + "response": [], + "data": str(base64.b64encode(bytes(response.getvalue(), "utf-8")), "utf-8"), + } def introspection(): @@ -98,7 +111,7 @@ def introspection(): "responseType": "application/txt", "outputFileExtension": "csv", "userConfig": {}, - "inputSource": [] + "inputSource": [], } return modulesetup diff --git a/misp_modules/modules/export_mod/threat_connect_export.py b/misp_modules/modules/export_mod/threat_connect_export.py index a86240c0d..ea61ca66c 100644 --- a/misp_modules/modules/export_mod/threat_connect_export.py +++ b/misp_modules/modules/export_mod/threat_connect_export.py @@ -4,6 +4,7 @@ Source: http://kb.threatconnect.com/customer/en/portal/articles/1912599-using-structured-import/ Source: http://kb.threatconnect.com/customer/en/portal/articles/2092925-the-threatconnect-data-model/ """ + import base64 import csv import io @@ -13,17 +14,21 @@ misperrors = {"error": "Error"} moduleinfo = { - 'version': '0.1', - 'author': 'CenturyLink CIRT', - 'description': 'Module to export a structured CSV file for uploading to ThreatConnect.', - 'module-type': ['export'], - 'name': 'ThreadConnect Export', - 'logo': 'threatconnect.png', - 'requirements': ['csv'], - 'features': 'The module takes a MISP event in input, to look every attribute. Each attribute matching with some predefined types is then exported in a CSV format recognized by ThreatConnect.\nUsers should then provide, as module configuration, the source of data they export, because it is required by the output format.', - 'references': ['https://www.threatconnect.com'], - 'input': 'MISP Event attributes', - 'output': 'ThreatConnect CSV format file', + "version": "0.1", + "author": "CenturyLink CIRT", + "description": "Module to export a structured CSV file for uploading to ThreatConnect.", + "module-type": ["export"], + "name": "ThreadConnect Export", + "logo": "threatconnect.png", + "requirements": ["csv"], + "features": ( + "The module takes a MISP event in input, to look every attribute. Each attribute matching with some predefined" + " types is then exported in a CSV format recognized by ThreatConnect.\nUsers should then provide, as module" + " configuration, the source of data they export, because it is required by the output format." + ), + "references": ["https://www.threatconnect.com"], + "input": "MISP Event attributes", + "output": "ThreatConnect CSV format file", } # config fields expected from the MISP administrator @@ -44,13 +49,11 @@ "email-dst": "EmailAddress", "url": "URL", "md5": "File", - "filename|md5": "File" + "filename|md5": "File", } # combine all the MISP fields from fieldmap into one big list -mispattributes = { - "input": list(fieldmap.keys()) -} +mispattributes = {"input": list(fieldmap.keys())} def handler(q=False): @@ -75,27 +78,37 @@ def handler(q=False): for event in request["data"]: for attribute in event["Attribute"]: if attribute["type"] in mispattributes["input"]: - logging.debug("Adding %s to structured CSV export of ThreatConnectExport", attribute["value"]) + logging.debug( + "Adding %s to structured CSV export of ThreatConnectExport", + attribute["value"], + ) if "|" in attribute["type"]: # if the attribute type has multiple values, line it up with the corresponding ThreatConnect values in fieldmap indicators = tuple(attribute["value"].split("|")) tc_types = tuple(fieldmap[attribute["type"]].split("|")) for i, indicator in enumerate(indicators): - writer.writerow({ - "Type": tc_types[i], - "Value": indicator, - "Source": config["Default_Source"], - "Description": attribute["comment"] - }) + writer.writerow( + { + "Type": tc_types[i], + "Value": indicator, + "Source": config["Default_Source"], + "Description": attribute["comment"], + } + ) else: - writer.writerow({ - "Type": fieldmap[attribute["type"]], - "Value": attribute["value"], - "Source": config["Default_Source"], - "Description": attribute["comment"] - }) + writer.writerow( + { + "Type": fieldmap[attribute["type"]], + "Value": attribute["value"], + "Source": config["Default_Source"], + "Description": attribute["comment"], + } + ) - return {"response": [], "data": str(base64.b64encode(bytes(response.getvalue(), 'utf-8')), 'utf-8')} + return { + "response": [], + "data": str(base64.b64encode(bytes(response.getvalue(), "utf-8")), "utf-8"), + } def introspection(): @@ -109,7 +122,7 @@ def introspection(): "responseType": "application/txt", "outputFileExtension": "csv", "userConfig": {}, - "inputSource": [] + "inputSource": [], } return modulesetup diff --git a/misp_modules/modules/export_mod/virustotal_collections.py b/misp_modules/modules/export_mod/virustotal_collections.py index 9da3bb5de..bf9702d12 100644 --- a/misp_modules/modules/export_mod/virustotal_collections.py +++ b/misp_modules/modules/export_mod/virustotal_collections.py @@ -17,119 +17,117 @@ import base64 import json + import requests -misperrors = { - 'error': 'Error' -} +misperrors = {"error": "Error"} mispattributes = { - 'input': [ - 'hostname', - 'domain', - 'ip-src', - 'ip-dst', - 'md5', - 'sha1', - 'sha256', - 'url' - ], - 'format': 'misp_standard', - 'responseType': 'application/txt', - 'outputFileExtension': 'txt', + "input": ["hostname", "domain", "ip-src", "ip-dst", "md5", "sha1", "sha256", "url"], + "format": "misp_standard", + "responseType": "application/txt", + "outputFileExtension": "txt", } moduleinfo = { - 'version': '1.0', - 'author': 'VirusTotal', - 'description': 'Creates a VT Collection from an event iocs.', - 'module-type': ['export'], - 'name': 'VirusTotal Collections Export', - 'logo': 'virustotal.png', - 'requirements': ['An access to the VirusTotal API (apikey).'], - 'features': 'This export module which takes advantage of a new endpoint in VT APIv3 to create VT Collections from IOCs contained in a MISP event. With this module users will be able to create a collection just using the Download as... button.', - 'references': ['https://www.virustotal.com/', 'https://blog.virustotal.com/2021/11/introducing-virustotal-collections.html'], - 'input': 'A domain, hash (md5, sha1, sha256 or sha512), hostname, url or IP address attribute.', - 'output': 'A VirusTotal collection in VT.', + "version": "1.0", + "author": "VirusTotal", + "description": "Creates a VT Collection from an event iocs.", + "module-type": ["export"], + "name": "VirusTotal Collections Export", + "logo": "virustotal.png", + "requirements": ["An access to the VirusTotal API (apikey)."], + "features": ( + "This export module which takes advantage of a new endpoint in VT APIv3 to create VT Collections from IOCs" + " contained in a MISP event. With this module users will be able to create a collection just using the Download" + " as... button." + ), + "references": [ + "https://www.virustotal.com/", + "https://blog.virustotal.com/2021/11/introducing-virustotal-collections.html", + ], + "input": "A domain, hash (md5, sha1, sha256 or sha512), hostname, url or IP address attribute.", + "output": "A VirusTotal collection in VT.", } moduleconfig = [ - 'vt_api_key', - 'proxy_host', - 'proxy_port', - 'proxy_username', - 'proxy_password' + "vt_api_key", + "proxy_host", + "proxy_port", + "proxy_username", + "proxy_password", ] class VTError(Exception): - "Exception class to map vt api response errors." - pass + "Exception class to map vt api response errors." + + pass def create_collection(api_key, event_data): - headers = { - 'x-apikey': api_key, - 'content-type': 'application/json', - 'x-tool': 'MISPModuleVirusTotalCollectionExport', - } + headers = { + "x-apikey": api_key, + "content-type": "application/json", + "x-tool": "MISPModuleVirusTotalCollectionExport", + } - response = requests.post('https://www.virustotal.com/api/v3/integrations/misp/collections', - headers=headers, - json=event_data) + response = requests.post( + "https://www.virustotal.com/api/v3/integrations/misp/collections", + headers=headers, + json=event_data, + ) - uuid = event_data['Event']['uuid'] - response_data = response.json() + uuid = event_data["Event"]["uuid"] + response_data = response.json() - if response.status_code == 200: - col_id = response_data['data']['id'] - return f'{uuid}: https://www.virustotal.com/gui/collection/{col_id}/iocs' + if response.status_code == 200: + col_id = response_data["data"]["id"] + return f"{uuid}: https://www.virustotal.com/gui/collection/{col_id}/iocs" - error = response_data['error']['message'] - if response.status_code == 400: - return f'{uuid}: {error}' - else: - misperrors['error'] = error - raise VTError(error) + error = response_data["error"]["message"] + if response.status_code == 400: + return f"{uuid}: {error}" + else: + misperrors["error"] = error + raise VTError(error) def normalize_misp_data(data): - normalized_data = {'Event': data.pop('Event', {})} - for attr_key in data: - if isinstance(data[attr_key], list) or isinstance(data[attr_key], dict): - if attr_key == 'EventTag': - normalized_data['Event']['Tag'] = [tag['Tag'] for tag in data[attr_key]] - else: - normalized_data['Event'][attr_key] = data[attr_key] + normalized_data = {"Event": data.pop("Event", {})} + for attr_key in data: + if isinstance(data[attr_key], list) or isinstance(data[attr_key], dict): + if attr_key == "EventTag": + normalized_data["Event"]["Tag"] = [tag["Tag"] for tag in data[attr_key]] + else: + normalized_data["Event"][attr_key] = data[attr_key] - return normalized_data + return normalized_data def handler(q=False): - request = json.loads(q) - - if not request.get('config') or not request['config'].get('vt_api_key'): - misperrors['error'] = 'A VirusTotal api key is required for this module.' - return misperrors - - config = request['config'] - data = request['data'] - responses = [] - - try: - for event_data in data: - normalized_event = normalize_misp_data(event_data) - responses.append(create_collection(config.get('vt_api_key'), - normalized_event)) - - output = '\n'.join(responses) - return { - "response": [], - "data": str(base64.b64encode(bytes(output, 'utf-8')), 'utf-8'), - } - except VTError: - return misperrors + request = json.loads(q) + + if not request.get("config") or not request["config"].get("vt_api_key"): + misperrors["error"] = "A VirusTotal api key is required for this module." + return misperrors + + config = request["config"] + data = request["data"] + responses = [] + + try: + for event_data in data: + normalized_event = normalize_misp_data(event_data) + responses.append(create_collection(config.get("vt_api_key"), normalized_event)) + output = "\n".join(responses) + return { + "response": [], + "data": str(base64.b64encode(bytes(output, "utf-8")), "utf-8"), + } + except VTError: + return misperrors def introspection(): @@ -137,5 +135,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/export_mod/vt_graph.py b/misp_modules/modules/export_mod/vt_graph.py index 7f2125c2a..45a9104b6 100644 --- a/misp_modules/modules/export_mod/vt_graph.py +++ b/misp_modules/modules/export_mod/vt_graph.py @@ -1,51 +1,51 @@ -'''Export MISP event to VirusTotal Graph.''' - +"""Export MISP event to VirusTotal Graph.""" import base64 import json -from vt_graph_parser.importers.pymisp_response import from_pymisp_response +from vt_graph_parser.importers.pymisp_response import from_pymisp_response -misperrors = { - 'error': 'Error' -} +misperrors = {"error": "Error"} moduleinfo = { - 'version': '0.1', - 'author': 'VirusTotal', - 'description': 'This module is used to create a VirusTotal Graph from a MISP event.', - 'module-type': ['export'], - 'name': 'VirusTotal Graph Export', - 'logo': 'virustotal.png', - 'requirements': ['vt_graph_api, the python library to query the VirusTotal graph API'], - 'features': 'The module takes the MISP event as input and queries the VirusTotal Graph API to create a new graph out of the event.\n\nOnce the graph is ready, we get the url of it, which is returned so we can view it on VirusTotal.', - 'references': ['https://www.virustotal.com/gui/graph-overview'], - 'input': 'A MISP event.', - 'output': 'Link of the VirusTotal Graph created for the event.', + "version": "0.1", + "author": "VirusTotal", + "description": "This module is used to create a VirusTotal Graph from a MISP event.", + "module-type": ["export"], + "name": "VirusTotal Graph Export", + "logo": "virustotal.png", + "requirements": ["vt_graph_api, the python library to query the VirusTotal graph API"], + "features": ( + "The module takes the MISP event as input and queries the VirusTotal Graph API to create a new graph out of the" + " event.\n\nOnce the graph is ready, we get the url of it, which is returned so we can view it on VirusTotal." + ), + "references": ["https://www.virustotal.com/gui/graph-overview"], + "input": "A MISP event.", + "output": "Link of the VirusTotal Graph created for the event.", } mispattributes = { - 'input': [ - 'hostname', - 'domain', - 'ip-src', - 'ip-dst', - 'md5', - 'sha1', - 'sha256', - 'url', - 'filename|md5', - 'filename' + "input": [ + "hostname", + "domain", + "ip-src", + "ip-dst", + "md5", + "sha1", + "sha256", + "url", + "filename|md5", + "filename", ] } moduleconfig = [ - 'vt_api_key', - 'fetch_information', - 'private', - 'fetch_vt_enterprise', - 'expand_one_level', - 'user_editors', - 'user_viewers', - 'group_editors', - 'group_viewers' + "vt_api_key", + "fetch_information", + "private", + "fetch_vt_enterprise", + "expand_one_level", + "user_editors", + "user_viewers", + "group_editors", + "group_viewers", ] @@ -62,37 +62,43 @@ def handler(q=False): return False request = json.loads(q) - if not request.get('config') or not request['config'].get('vt_api_key'): - misperrors['error'] = 'A VirusTotal api key is required for this module.' + if not request.get("config") or not request["config"].get("vt_api_key"): + misperrors["error"] = "A VirusTotal api key is required for this module." return misperrors - config = request['config'] + config = request["config"] - api_key = config.get('vt_api_key') - fetch_information = config.get('fetch_information') or False - private = config.get('private') or False - fetch_vt_enterprise = config.get('fetch_vt_enterprise') or False - expand_one_level = config.get('expand_one_level') or False + api_key = config.get("vt_api_key") + fetch_information = config.get("fetch_information") or False + private = config.get("private") or False + fetch_vt_enterprise = config.get("fetch_vt_enterprise") or False + expand_one_level = config.get("expand_one_level") or False - user_editors = config.get('user_editors') + user_editors = config.get("user_editors") if user_editors: - user_editors = user_editors.split(',') - user_viewers = config.get('user_viewers') + user_editors = user_editors.split(",") + user_viewers = config.get("user_viewers") if user_viewers: - user_viewers = user_viewers.split(',') - group_editors = config.get('group_editors') + user_viewers = user_viewers.split(",") + group_editors = config.get("group_editors") if group_editors: - group_editors = group_editors.split(',') - group_viewers = config.get('group_viewers') + group_editors = group_editors.split(",") + group_viewers = config.get("group_viewers") if group_viewers: - group_viewers = group_viewers.split(',') + group_viewers = group_viewers.split(",") graphs = from_pymisp_response( - request, api_key, fetch_information=fetch_information, - private=private, fetch_vt_enterprise=fetch_vt_enterprise, - user_editors=user_editors, user_viewers=user_viewers, - group_editors=group_editors, group_viewers=group_viewers, - expand_node_one_level=expand_one_level) + request, + api_key, + fetch_information=fetch_information, + private=private, + fetch_vt_enterprise=fetch_vt_enterprise, + user_editors=user_editors, + user_viewers=user_viewers, + group_editors=group_editors, + group_viewers=group_viewers, + expand_node_one_level=expand_one_level, + ) links = [] for graph in graphs: @@ -100,21 +106,20 @@ def handler(q=False): links.append(graph.get_ui_link()) # This file will contains one VirusTotal graph link for each exported event - file_data = str(base64.b64encode( - bytes('\n'.join(links), 'utf-8')), 'utf-8') - return {'response': [], 'data': file_data} + file_data = str(base64.b64encode(bytes("\n".join(links), "utf-8")), "utf-8") + return {"response": [], "data": file_data} def introspection(): modulesetup = { - 'responseType': 'application/txt', - 'outputFileExtension': 'txt', - 'userConfig': {}, - 'inputSource': [] + "responseType": "application/txt", + "outputFileExtension": "txt", + "userConfig": {}, + "inputSource": [], } return modulesetup def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/export_mod/yara_export.py b/misp_modules/modules/export_mod/yara_export.py index ebcbfcde2..84e421f4c 100644 --- a/misp_modules/modules/export_mod/yara_export.py +++ b/misp_modules/modules/export_mod/yara_export.py @@ -1,45 +1,44 @@ -import json import base64 +import json import re -try: - import yara -except (OSError, ImportError): - print("yara is missing, use 'pip3 install -I -r REQUIREMENTS' from the root of this repository to install it.") +import yara -misperrors = {'error': 'Error'} +misperrors = {"error": "Error"} -userConfig = { - -} +userConfig = {} moduleconfig = [] # fixed for now, options in the future: # event, attribute, event-collection, attribute-collection -inputSource = ['event'] +inputSource = ["event"] -outputFileExtension = 'yara' -responseType = 'text/plain' +outputFileExtension = "yara" +responseType = "text/plain" moduleinfo = { - 'version': '0.1', - 'author': 'Christophe Vandeplas', - 'description': 'This module is used to export MISP events to YARA.', - 'module-type': ['export'], - 'name': 'YARA Rule Export', - 'logo': 'yara.png', - 'requirements': ['yara-python python library'], - 'features': 'The module will dynamically generate YARA rules for attributes that are marked as to IDS. Basic metadata about the event is added to the rule.\nAttributes that are already YARA rules are also exported, with a rewritten rule name.', - 'references': ['https://virustotal.github.io/yara/'], - 'input': 'Attributes and Objects.', - 'output': 'A YARA file that can be used with the YARA scanning tool.', + "version": "0.1", + "author": "Christophe Vandeplas", + "description": "This module is used to export MISP events to YARA.", + "module-type": ["export"], + "name": "YARA Rule Export", + "logo": "yara.png", + "requirements": ["yara-python python library"], + "features": ( + "The module will dynamically generate YARA rules for attributes that are marked as to IDS. Basic metadata about" + " the event is added to the rule.\nAttributes that are already YARA rules are also exported, with a rewritten" + " rule name." + ), + "references": ["https://virustotal.github.io/yara/"], + "input": "Attributes and Objects.", + "output": "A YARA file that can be used with the YARA scanning tool.", } -class YaraRule(): +class YaraRule: def __init__(self, name): self.name = name self.strings = {} @@ -47,7 +46,7 @@ def __init__(self, name): self.meta = {} def add_string(self, type_: str, s: str): - type_clean = ''.join(c if c.isalnum() or c == '_' else '_' for c in type_) + type_clean = "".join(c if c.isalnum() or c == "_" else "_" for c in type_) if type_clean not in self.strings: self.strings[type_clean] = [] self.strings[type_clean].append(s) @@ -71,17 +70,17 @@ def __str__(self): for key, values in self.meta.items(): i = 0 if len(values) == 1: - result.append(f" {key} = \"{values[0]}\"") + result.append(f' {key} = "{values[0]}"') continue for value in values: - result.append(f" {key}_{i} = \"{value}\"") + result.append(f' {key}_{i} = "{value}"') i += 1 result.append(" strings:") for key, values in self.strings.items(): i = 0 for value in values: - result.append(f" ${key}_{i} = \"{value}\"") + result.append(f' ${key}_{i} = "{value}"') i += 1 result.append(" condition:") @@ -92,21 +91,21 @@ def __str__(self): result.append("}") result.append("") - return '\n'.join(result) + return "\n".join(result) def handle_string(yara_rules: list, yr: YaraRule, attribute: dict): - if not attribute['to_ids']: # skip non IDS attributes + if not attribute["to_ids"]: # skip non IDS attributes return - yr.add_string(attribute['type'], attribute['value']) + yr.add_string(attribute["type"], attribute["value"]) return def handle_combined(yara_rules: list, yr: YaraRule, attribute: dict): - if not attribute['to_ids']: # skip non IDS attributes + if not attribute["to_ids"]: # skip non IDS attributes return - type_1, type_2 = attribute['type'].split('|') - value_1, value_2 = attribute['value'].split('|') + type_1, type_2 = attribute["type"].split("|") + value_1, value_2 = attribute["value"].split("|") try: handlers[type_1](yara_rules, yr, type_1, value_1) except KeyError: @@ -122,17 +121,23 @@ def handle_combined(yara_rules: list, yr: YaraRule, attribute: dict): def handle_yara(yara_rules: list, yr: YaraRule, attribute: dict): # do not check for to_ids, as we want to always export the Yara rule # split out as a separate rule, and rewrite the rule name - value = re.sub('^[ \t]*rule ', 'rule MISP_e{}_'.format(attribute['event_id']), attribute['value'], flags=re.MULTILINE) + value = re.sub( + "^[ \t]*rule ", + "rule MISP_e{}_".format(attribute["event_id"]), + attribute["value"], + flags=re.MULTILINE, + ) # cleanup dirty stuff from people - substitutions = (('”', '"'), - ('“', '"'), - ('″', '"'), - ('`', "'"), - ('\r', ''), - ('Rule ', 'rule ') # some people write this with the wrong case - # ('$ ', '$'), # this breaks rules - # ('\t\t', '\n'), # this breaks rules - ) + substitutions = ( + ("”", '"'), + ("“", '"'), + ("″", '"'), + ("`", "'"), + ("\r", ""), + ("Rule ", "rule "), # some people write this with the wrong case + # ('$ ', '$'), # this breaks rules + # ('\t\t', '\n'), # this breaks rules + ) for substitution in substitutions: if substitution[0] in value: value = value.replace(substitution[0], substitution[1]) @@ -143,10 +148,15 @@ def handle_yara(yara_rules: list, yr: YaraRule, attribute: dict): # return # private rules need some more rewriting - if 'private rule' in value: - priv_rules = re.findall(r'private rule (\w+)', value, flags=re.MULTILINE) + if "private rule" in value: + priv_rules = re.findall(r"private rule (\w+)", value, flags=re.MULTILINE) for priv_rule in priv_rules: - value = re.sub(priv_rule, 'MISP_e{}_{}'.format(attribute['event_id'], priv_rule), value, flags=re.MULTILINE) + value = re.sub( + priv_rule, + "MISP_e{}_{}".format(attribute["event_id"], priv_rule), + value, + flags=re.MULTILINE, + ) # compile the yara rule to confirm it's validity try: @@ -161,64 +171,66 @@ def handle_yara(yara_rules: list, yr: YaraRule, attribute: dict): def handle_malware_sample(yara_rules: list, yr: YaraRule, attribute: dict): - if not attribute['to_ids']: # skip non IDS attributes + if not attribute["to_ids"]: # skip non IDS attributes return - handle_combined(yara_rules, yr, 'filename|md5', attribute['value']) + handle_combined(yara_rules, yr, "filename|md5", attribute["value"]) def handle_meta(yara_rules: list, yr: YaraRule, attribute: dict): - yr.add_meta(attribute['type'], attribute['value']) + yr.add_meta(attribute["type"], attribute["value"]) return handlers = { - 'yara': handle_yara, - 'hostname': handle_string, - 'hostname|port': handle_combined, - 'domain': handle_string, - 'domain|ip': handle_combined, - 'ip': handle_string, - 'ip-src': handle_string, - 'ip-dst': handle_string, - 'ip-dst|port': handle_combined, # we could also handle_string, which would be more specific. Less false positives, but less true positives too... - 'ip-src|port': handle_combined, - 'url': handle_string, - 'email': handle_string, - 'email-src': handle_string, - 'email-dst': handle_string, - 'email-subject': handle_string, - 'email-attachment': handle_string, - 'email-header': handle_string, - 'email-reply-to': handle_string, - 'email-x-mailer': handle_string, - 'email-mime-boundary': handle_string, - 'email-thread-index': handle_string, - 'email-message-id': handle_string, - 'filename': handle_string, - 'filename|md5': handle_combined, - 'filename|sha1': handle_combined, - 'filename|sha256': handle_combined, - 'filename|authentihash': handle_combined, - 'filename|vhash': handle_combined, - 'filename|ssdeep': handle_combined, - 'filename|imphash': handle_combined, - 'filename|impfuzzy': handle_combined, - 'filename|pehash': handle_combined, - 'filename|sha224': handle_combined, - 'filename|sha384': handle_combined, - 'filename|sha512': handle_combined, - 'filename|sha512/224': handle_combined, - 'filename|sha512/256': handle_combined, - 'filename|sha3-224': handle_combined, - 'filename|sha3-256': handle_combined, - 'filename|sha3-384': handle_combined, - 'filename|sha3-512': handle_combined, - 'filename|tlsh': handle_combined, - 'malware-sample': handle_malware_sample, - 'pattern-in-file': handle_string, - 'pattern-in-traffic': handle_string, - 'pattern-in-memory': handle_string, - 'link': handle_meta + "yara": handle_yara, + "hostname": handle_string, + "hostname|port": handle_combined, + "domain": handle_string, + "domain|ip": handle_combined, + "ip": handle_string, + "ip-src": handle_string, + "ip-dst": handle_string, + "ip-dst|port": ( + handle_combined + ), # we could also handle_string, which would be more specific. Less false positives, but less true positives too... + "ip-src|port": handle_combined, + "url": handle_string, + "email": handle_string, + "email-src": handle_string, + "email-dst": handle_string, + "email-subject": handle_string, + "email-attachment": handle_string, + "email-header": handle_string, + "email-reply-to": handle_string, + "email-x-mailer": handle_string, + "email-mime-boundary": handle_string, + "email-thread-index": handle_string, + "email-message-id": handle_string, + "filename": handle_string, + "filename|md5": handle_combined, + "filename|sha1": handle_combined, + "filename|sha256": handle_combined, + "filename|authentihash": handle_combined, + "filename|vhash": handle_combined, + "filename|ssdeep": handle_combined, + "filename|imphash": handle_combined, + "filename|impfuzzy": handle_combined, + "filename|pehash": handle_combined, + "filename|sha224": handle_combined, + "filename|sha384": handle_combined, + "filename|sha512": handle_combined, + "filename|sha512/224": handle_combined, + "filename|sha512/256": handle_combined, + "filename|sha3-224": handle_combined, + "filename|sha3-256": handle_combined, + "filename|sha3-384": handle_combined, + "filename|sha3-512": handle_combined, + "filename|tlsh": handle_combined, + "malware-sample": handle_malware_sample, + "pattern-in-file": handle_string, + "pattern-in-traffic": handle_string, + "pattern-in-memory": handle_string, + "link": handle_meta, } # auto-generate the list of types to use @@ -232,30 +244,33 @@ def handler(q=False): yara_rules = [] for event in request["data"]: - event_info_clean = ''.join(c if c.isalnum() or c == '_' else '_' for c in event['Event']['info']) + event_info_clean = "".join(c if c.isalnum() or c == "_" else "_" for c in event["Event"]["info"]) yr = YaraRule(f"MISP_e{event['Event']['id']}_{event_info_clean}") - yr.add_meta('description', event['Event']['info']) - yr.add_meta('author', f"MISP - {event['Orgc']['name']}") - yr.add_meta('misp_event_date', event['Event']['date']) - yr.add_meta('misp_event_id', event['Event']['id']) - yr.add_meta('misp_event_uuid', event['Event']['uuid']) + yr.add_meta("description", event["Event"]["info"]) + yr.add_meta("author", f"MISP - {event['Orgc']['name']}") + yr.add_meta("misp_event_date", event["Event"]["date"]) + yr.add_meta("misp_event_id", event["Event"]["id"]) + yr.add_meta("misp_event_uuid", event["Event"]["uuid"]) for attribute in event.get("Attribute", []): try: - handlers[attribute['type']](yara_rules, yr, attribute) + handlers[attribute["type"]](yara_rules, yr, attribute) except KeyError: # ignore unsupported types pass for obj in event.get("Object", []): for attribute in obj["Attribute"]: try: - handlers[attribute['type']](yara_rules, yr, attribute) + handlers[attribute["type"]](yara_rules, yr, attribute) except KeyError: # ignore unsupported types pass yara_rules.append(str(yr)) - r = {"response": [], "data": str(base64.b64encode(bytes('\n'.join(yara_rules), 'utf-8')), 'utf-8')} + r = { + "response": [], + "data": str(base64.b64encode(bytes("\n".join(yara_rules), "utf-8")), "utf-8"), + } return r @@ -264,27 +279,27 @@ def introspection(): modulesetup = {} try: responseType - modulesetup['responseType'] = responseType + modulesetup["responseType"] = responseType except NameError: pass try: userConfig - modulesetup['userConfig'] = userConfig + modulesetup["userConfig"] = userConfig except NameError: pass try: outputFileExtension - modulesetup['outputFileExtension'] = outputFileExtension + modulesetup["outputFileExtension"] = outputFileExtension except NameError: pass try: inputSource - modulesetup['inputSource'] = inputSource + modulesetup["inputSource"] = inputSource except NameError: pass return modulesetup def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/import_mod/__init__.py b/misp_modules/modules/import_mod/__init__.py index 9ae118d28..e69de29bb 100644 --- a/misp_modules/modules/import_mod/__init__.py +++ b/misp_modules/modules/import_mod/__init__.py @@ -1,22 +0,0 @@ -import os -import sys -sys.path.append('{}/lib'.format('/'.join((os.path.realpath(__file__)).split('/')[:-3]))) - -__all__ = [ - 'vmray_import', - 'lastline_import', - 'ocr', - 'cuckooimport', - 'goamlimport', - 'email_import', - 'mispjson', - 'openiocimport', - 'threatanalyzer_import', - 'csvimport', - 'cof2misp', - 'joe_import', - 'taxii21', - 'url_import', - 'vmray_summary_json_import', - 'import_blueprint' -] diff --git a/misp_modules/modules/import_mod/cof2misp.py b/misp_modules/modules/import_mod/cof2misp.py index 794265449..bbcd47511 100755 --- a/misp_modules/modules/import_mod/cof2misp.py +++ b/misp_modules/modules/import_mod/cof2misp.py @@ -1,4 +1,4 @@ -""" PassiveDNS Common Output Format (COF) MISP importer. +"""PassiveDNS Common Output Format (COF) MISP importer. Takes as input a valid COF file or the output of the dnsdbflex utility and creates MISP objects for the input. @@ -12,43 +12,45 @@ """ -import sys -import json import base64 - +import json +import sys import ndjson +from cof2misp.cof import validate_cof, validate_dnsdbflex # from pymisp import MISPObject, MISPEvent, PyMISP from pymisp import MISPObject -from cof2misp.cof import validate_cof, validate_dnsdbflex - +create_specific_attributes = False # this is for https://github.com/MISP/misp-objects/pull/314 -create_specific_attributes = False # this is for https://github.com/MISP/misp-objects/pull/314 - -misperrors = {'error': 'Error'} +misperrors = {"error": "Error"} userConfig = {} -inputSource = ['file'] +inputSource = ["file"] -mispattributes = {'inputSource': ['file'], 'output': ['MISP objects'], - 'format': 'misp_standard'} +mispattributes = { + "inputSource": ["file"], + "output": ["MISP objects"], + "format": "misp_standard", +} moduleinfo = { - 'version': '0.3', - 'author': 'Aaron Kaplan', - 'description': 'Passive DNS Common Output Format (COF) MISP importer', - 'module-type': ['import'], - 'name': 'PDNS COF Importer', - 'requirements': ['PyMISP'], - 'features': 'Takes as input a valid COF file or the output of the dnsdbflex utility and creates MISP objects for the input.', - 'references': ['https://tools.ietf.org/id/draft-dulaunoy-dnsop-passive-dns-cof-08.html'], - 'input': 'Passive DNS output in Common Output Format (COF)', - 'output': 'MISP objects', - 'logo': '', + "version": "0.3", + "author": "Aaron Kaplan", + "description": "Passive DNS Common Output Format (COF) MISP importer", + "module-type": ["import"], + "name": "PDNS COF Importer", + "requirements": ["PyMISP"], + "features": ( + "Takes as input a valid COF file or the output of the dnsdbflex utility and creates MISP objects for the input." + ), + "references": ["https://tools.ietf.org/id/draft-dulaunoy-dnsop-passive-dns-cof-08.html"], + "input": "Passive DNS output in Common Output Format (COF)", + "output": "MISP objects", + "logo": "", } moduleconfig = [] @@ -76,58 +78,67 @@ def parse_and_insert_cof(data: str) -> dict: objects = [] try: entries = ndjson.loads(data) - for entry in entries: # iterate over all ndjson lines + for entry in entries: # iterate over all ndjson lines # validate here (simple validation or full JSON Schema validation) if not validate_cof(entry): return {"error": "Could not validate the COF input '%s'" % entry} # Next, extract some fields - rrtype = entry['rrtype'].upper() - rrname = entry['rrname'].rstrip('.') - rdata = [x.rstrip('.') for x in entry['rdata']] + rrtype = entry["rrtype"].upper() + rrname = entry["rrname"].rstrip(".") + rdata = [x.rstrip(".") for x in entry["rdata"]] # create a new MISP object, based on the passive-dns object for each nd-JSON line - o = MISPObject(name='passive-dns', standalone=False, comment='created by cof2misp') + o = MISPObject(name="passive-dns", standalone=False, comment="created by cof2misp") # o.add_tag('tlp:amber') # FIXME: we'll want to add a tlp: tag to the object - if 'bailiwick' in entry: - o.add_attribute('bailiwick', value=entry['bailiwick'].rstrip('.'), distribution=0) + if "bailiwick" in entry: + o.add_attribute("bailiwick", value=entry["bailiwick"].rstrip("."), distribution=0) # # handle the combinations of rrtype (domain, ip) on both left and right side # if create_specific_attributes: - if rrtype in ['A', 'AAAA', 'A6']: # address type + if rrtype in ["A", "AAAA", "A6"]: # address type # address type - o.add_attribute('rrname_domain', value=rrname, distribution=0) + o.add_attribute("rrname_domain", value=rrname, distribution=0) for r in rdata: - o.add_attribute('rdata_ip', value=r, distribution=0) - elif rrtype in ['CNAME', 'DNAME', 'NS']: # both sides are domains - o.add_attribute('rrname_domain', value=rrname, distribution=0) + o.add_attribute("rdata_ip", value=r, distribution=0) + elif rrtype in ["CNAME", "DNAME", "NS"]: # both sides are domains + o.add_attribute("rrname_domain", value=rrname, distribution=0) for r in rdata: - o.add_attribute('rdata_domain', value=r, distribution=0) - elif rrtype in ['SOA']: # left side is a domain, right side is text - o.add_attribute('rrname_domain', value=rrname, distribution=0) + o.add_attribute("rdata_domain", value=r, distribution=0) + elif rrtype in ["SOA"]: # left side is a domain, right side is text + o.add_attribute("rrname_domain", value=rrname, distribution=0) # # now do the regular filling up of rrname, rrtype, time_first, etc. # - o.add_attribute('rrname', value=rrname, distribution=0) - o.add_attribute('rrtype', value=rrtype, distribution=0) + o.add_attribute("rrname", value=rrname, distribution=0) + o.add_attribute("rrtype", value=rrtype, distribution=0) for r in rdata: - o.add_attribute('rdata', value=r, distribution=0) - o.add_attribute('raw_rdata', value=json.dumps(rdata), distribution=0) # FIXME: do we need to hex encode it? - o.add_attribute('time_first', value=entry['time_first'], distribution=0) - o.add_attribute('time_last', value=entry['time_last'], distribution=0) - o.first_seen = entry['time_first'] # is this redundant? - o.last_seen = entry['time_last'] + o.add_attribute("rdata", value=r, distribution=0) + o.add_attribute("raw_rdata", value=json.dumps(rdata), distribution=0) # FIXME: do we need to hex encode it? + o.add_attribute("time_first", value=entry["time_first"], distribution=0) + o.add_attribute("time_last", value=entry["time_last"], distribution=0) + o.first_seen = entry["time_first"] # is this redundant? + o.last_seen = entry["time_last"] # # Now add the other optional values. # FIXME: how about a map() other function. DNRY # - for k in ['count', 'sensor_id', 'origin', 'text', 'time_first_ms', 'time_last_ms', 'zone_time_first', 'zone_time_last']: + for k in [ + "count", + "sensor_id", + "origin", + "text", + "time_first_ms", + "time_last_ms", + "zone_time_first", + "zone_time_last", + ]: if k in entry and entry[k]: o.add_attribute(k, value=entry[k], distribution=0) @@ -136,7 +147,7 @@ def parse_and_insert_cof(data: str) -> dict: # objects.append(o.to_json()) - r = {'results': {'Object': [json.loads(o) for o in objects]}} + r = {"results": {"Object": [json.loads(o) for o in objects]}} except Exception as ex: misperrors["error"] = "An error occured during parsing of input: '%s'" % (str(ex),) return misperrors @@ -161,20 +172,35 @@ def parse_and_insert_dnsdbflex(data: str): objects = [] try: entries = ndjson.loads(data) - for entry in entries: # iterate over all ndjson lines + for entry in entries: # iterate over all ndjson lines # validate here (simple validation or full JSON Schema validation) if not validate_dnsdbflex(entry): return {"error": "Could not validate the dnsdbflex input '%s'" % entry} # Next, extract some fields - rrtype = entry['rrtype'].upper() - rrname = entry['rrname'].rstrip('.') + rrtype = entry["rrtype"].upper() + rrname = entry["rrname"].rstrip(".") # create a new MISP object, based on the passive-dns object for each nd-JSON line try: - o = MISPObject(name='passive-dns', standalone=False, distribution=0, comment='DNSDBFLEX import by cof2misp') - o.add_attribute('rrtype', value=rrtype, distribution=0, comment='DNSDBFLEX import by cof2misp') - o.add_attribute('rrname', value=rrname, distribution=0, comment='DNSDBFLEX import by cof2misp') + o = MISPObject( + name="passive-dns", + standalone=False, + distribution=0, + comment="DNSDBFLEX import by cof2misp", + ) + o.add_attribute( + "rrtype", + value=rrtype, + distribution=0, + comment="DNSDBFLEX import by cof2misp", + ) + o.add_attribute( + "rrname", + value=rrname, + distribution=0, + comment="DNSDBFLEX import by cof2misp", + ) except Exception as ex: print("could not create object. Reason: %s" % str(ex)) @@ -183,7 +209,7 @@ def parse_and_insert_dnsdbflex(data: str): # objects.append(o.to_json()) - r = {'results': {'Object': [json.loads(o) for o in objects]}} + r = {"results": {"Object": [json.loads(o) for o in objects]}} except Exception as ex: misperrors["error"] = "An error occured during parsing of input: '%s'" % (str(ex),) return misperrors @@ -209,11 +235,14 @@ def is_dnsdbflex(data: str) -> bool: try: j = ndjson.loads(data) for line in j: - if not set(line.keys()) == {'rrname', 'rrtype'}: - return False # shortcut. We assume it's not if a single line does not conform + if not set(line.keys()) == {"rrname", "rrtype"}: + return False # shortcut. We assume it's not if a single line does not conform return True except Exception as ex: - print("oops, this should not have happened. Maybe not an ndjson file? Reason: %s" % (str(ex),), file=sys.stderr) + print( + "oops, this should not have happened. Maybe not an ndjson file? Reason: %s" % (str(ex),), + file=sys.stderr, + ) return False @@ -234,19 +263,24 @@ def handler(q=False): # event = misp.get_event(event_id) # print("event_id = %s" % event_id, file=sys.stderr) try: - data = base64.b64decode(request["data"]).decode('utf-8') + data = base64.b64decode(request["data"]).decode("utf-8") if not data: - return json.dumps({'success': 0}) # empty file is ok + return json.dumps({"success": 0}) # empty file is ok if is_dnsdbflex(data): return parse_and_insert_dnsdbflex(data) elif is_cof(data): # check if it's valid COF format return parse_and_insert_cof(data) else: - return {'error': 'Could not find any valid COF input nor dnsdbflex input. Please have a loot at: https://datatracker.ietf.org/doc/draft-dulaunoy-dnsop-passive-dns-cof/'} + return { + "error": ( + "Could not find any valid COF input nor dnsdbflex input. Please have a loot at:" + " https://datatracker.ietf.org/doc/draft-dulaunoy-dnsop-passive-dns-cof/" + ) + } except Exception as ex: print("oops, got exception %s" % str(ex), file=sys.stderr) - return {'error': "Got exception %s" % str(ex)} + return {"error": "Got exception %s" % str(ex)} def introspection(): @@ -254,11 +288,11 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo -if __name__ == '__main__': - x = open('test.json', 'r') +if __name__ == "__main__": + x = open("test.json", "r") r = handler(q=x.read()) print(json.dumps(r)) diff --git a/misp_modules/modules/import_mod/csvimport.py b/misp_modules/modules/import_mod/csvimport.py index 52d3cff69..ff480deb7 100644 --- a/misp_modules/modules/import_mod/csvimport.py +++ b/misp_modules/modules/import_mod/csvimport.py @@ -1,41 +1,92 @@ -from pymisp import MISPEvent, MISPObject +import base64 import csv import io -import base64 -misperrors = {'error': 'Error'} +from pymisp import MISPEvent, MISPObject + +misperrors = {"error": "Error"} moduleinfo = { - 'version': '0.2', - 'author': 'Christian Studer', - 'module-type': ['import'], - 'name': 'CSV Import', - 'description': 'Module to import MISP attributes from a csv file.', - 'requirements': ['PyMISP'], - 'features': "In order to parse data from a csv file, a header is required to let the module know which column is matching with known attribute fields / MISP types.\n\nThis header either comes from the csv file itself or is part of the configuration of the module and should be filled out in MISP plugin settings, each field separated by COMMAS. Fields that do not match with any type known in MISP or are not MISP attribute fields should be ignored in import, using a space or simply nothing between two separators (example: 'ip-src, , comment, ').\n\nIf the csv file already contains a header that does not start by a '#', you should tick the checkbox 'has_header' to avoid importing it and have potential issues. You can also redefine the header even if it is already contained in the file, by following the rules for headers explained earlier. One reason why you would redefine a header is for instance when you want to skip some fields, or some fields are not valid types.", - 'references': ['https://tools.ietf.org/html/rfc4180', 'https://tools.ietf.org/html/rfc7111'], - 'input': 'CSV format file.', - 'output': 'MISP Event attributes', - 'logo': '', + "version": "0.2", + "author": "Christian Studer", + "module-type": ["import"], + "name": "CSV Import", + "description": "Module to import MISP attributes from a csv file.", + "requirements": ["PyMISP"], + "features": ( + "In order to parse data from a csv file, a header is required to let the module know which column is matching" + " with known attribute fields / MISP types.\n\nThis header either comes from the csv file itself or is part of" + " the configuration of the module and should be filled out in MISP plugin settings, each field separated by" + " COMMAS. Fields that do not match with any type known in MISP or are not MISP attribute fields should be" + " ignored in import, using a space or simply nothing between two separators (example: 'ip-src, , comment," + " ').\n\nIf the csv file already contains a header that does not start by a '#', you should tick the checkbox" + " 'has_header' to avoid importing it and have potential issues. You can also redefine the header even if it is" + " already contained in the file, by following the rules for headers explained earlier. One reason why you would" + " redefine a header is for instance when you want to skip some fields, or some fields are not valid types." + ), + "references": [ + "https://tools.ietf.org/html/rfc4180", + "https://tools.ietf.org/html/rfc7111", + ], + "input": "CSV format file.", + "output": "MISP Event attributes", + "logo": "", } moduleconfig = [] userConfig = { - 'header': { - 'type': 'String', - 'message': 'Define the header of the csv file, with types (included in MISP attribute types or attribute fields) separated by commas.\nFor fields that do not match these types or that you want to skip, please use space or simply nothing between commas.\nFor instance: ip-src,domain, ,timestamp'}, - 'has_header': { - 'type': 'Boolean', - 'message': 'Tick this box ONLY if there is a header line, NOT COMMENTED, and all the fields of this header are respecting the recommendations above.'}, - 'special_delimiter': { - 'type': 'String', - 'message': 'IF THE DELIMITERS ARE NOT COMMAS, please specify which ones are used (for instance: ";", "|", "/", "\t" for tabs, etc).' - } + "header": { + "type": "String", + "message": ( + "Define the header of the csv file, with types (included in MISP attribute types or attribute fields)" + " separated by commas.\nFor fields that do not match these types or that you want to skip, please use space" + " or simply nothing between commas.\nFor instance: ip-src,domain, ,timestamp" + ), + }, + "has_header": { + "type": "Boolean", + "message": ( + "Tick this box ONLY if there is a header line, NOT COMMENTED, and all the fields of this header are" + " respecting the recommendations above." + ), + }, + "special_delimiter": { + "type": "String", + "message": ( + 'IF THE DELIMITERS ARE NOT COMMAS, please specify which ones are used (for instance: ";", "|", "/", "\t"' + " for tabs, etc)." + ), + }, +} +mispattributes = { + "userConfig": userConfig, + "inputSource": ["file"], + "format": "misp_standard", } -mispattributes = {'userConfig': userConfig, 'inputSource': ['file'], 'format': 'misp_standard'} -misp_standard_csv_header = ['uuid', 'event_id', 'category', 'type', 'value', 'comment', 'to_ids', 'date', - 'object_relation', 'attribute_tag', 'object_uuid', 'object_name', 'object_meta_category'] -misp_context_additional_fields = ['event_info', 'event_member_org', 'event_source_org', 'event_distribution', - 'event_threat_level_id', 'event_analysis', 'event_date', 'event_tag'] +misp_standard_csv_header = [ + "uuid", + "event_id", + "category", + "type", + "value", + "comment", + "to_ids", + "date", + "object_relation", + "attribute_tag", + "object_uuid", + "object_name", + "object_meta_category", +] +misp_context_additional_fields = [ + "event_info", + "event_member_org", + "event_source_org", + "event_distribution", + "event_threat_level_id", + "event_analysis", + "event_date", + "event_tag", +] misp_extended_csv_header = misp_standard_csv_header + misp_context_additional_fields @@ -50,14 +101,16 @@ def __init__(self, header, has_header, delimiter, data, from_misp, MISPtypes, ca self.MISPtypes = MISPtypes self.categories = categories self.fields_number = len(self.header) - self.__score_mapping = {0: self.__create_standard_attribute, - 1: self.__create_attribute_with_ids, - 2: self.__create_attribute_with_tags, - 3: self.__create_attribute_with_ids_and_tags, - 4: self.__create_attribute_check_category, - 5: self.__create_attribute_check_category_and_ids, - 6: self.__create_attribute_check_category_and_tags, - 7: self.__create_attribute_check_category_with_ids_and_tags} + self.__score_mapping = { + 0: self.__create_standard_attribute, + 1: self.__create_attribute_with_ids, + 2: self.__create_attribute_with_tags, + 3: self.__create_attribute_with_ids_and_tags, + 4: self.__create_attribute_check_category, + 5: self.__create_attribute_check_category_and_ids, + 6: self.__create_attribute_check_category_and_tags, + 7: self.__create_attribute_check_category_with_ids_and_tags, + } def parse_csv(self): if self.from_misp: @@ -65,7 +118,7 @@ def parse_csv(self): self.__parse_misp_csv() else: attribute_fields = misp_standard_csv_header[:1] + misp_standard_csv_header[2:10] - object_fields = ['object_id'] + misp_standard_csv_header[10:] + object_fields = ["object_id"] + misp_standard_csv_header[10:] attribute_indexes = [] object_indexes = [] for i in range(len(self.header)): @@ -74,13 +127,27 @@ def parse_csv(self): elif self.header[i] in object_fields: object_indexes.append(i) if object_indexes: - if not any(field in self.header for field in ('object_uuid', 'object_id')) or 'object_name' not in self.header: + if ( + not any(field in self.header for field in ("object_uuid", "object_id")) + or "object_name" not in self.header + ): for line in self.data: for index in object_indexes: if line[index].strip(): - return {'error': 'It is not possible to import MISP objects from your csv file if you do not specify any object identifier and object name to separate each object from each other.'} - if 'object_relation' not in self.header: - return {'error': 'In order to import MISP objects, an object relation for each attribute contained in an object is required.'} + return { + "error": ( + "It is not possible to import MISP objects from your csv file if you do not" + " specify any object identifier and object name to separate each object" + " from each other." + ) + } + if "object_relation" not in self.header: + return { + "error": ( + "In order to import MISP objects, an object relation for each attribute contained in an" + " object is required." + ) + } self.__build_misp_event(attribute_indexes, object_indexes) else: attribute_fields = misp_standard_csv_header[:1] + misp_standard_csv_header[2:9] @@ -93,7 +160,7 @@ def parse_csv(self): types_indexes.append(i) self.__parse_external_csv(attribute_indexes, types_indexes) self.__finalize_results() - return {'success': 1} + return {"success": 1} ################################################################################ # Parsing csv data with MISP fields, # @@ -104,16 +171,16 @@ def __build_misp_event(self, attribute_indexes, object_indexes): score = self.__get_score() if object_indexes: objects = {} - id_name = 'object_id' if 'object_id' in self.header else 'object_uuid' + id_name = "object_id" if "object_id" in self.header else "object_uuid" object_id_index = self.header.index(id_name) - name_index = self.header.index('object_name') + name_index = self.header.index("object_name") for line in self.data: attribute = self.__score_mapping[score](line, attribute_indexes) object_id = line[object_id_index] if object_id: if object_id not in objects: misp_object = MISPObject(line[name_index]) - if id_name == 'object_uuid': + if id_name == "object_uuid": misp_object.uuid = object_id objects[object_id] = misp_object objects[object_id].add_attribute(**attribute) @@ -141,13 +208,13 @@ def __parse_external_csv(self, attribute_indexes, types_indexes): except IndexError: continue for index in types_indexes: - attribute = {'type': self.header[index], 'value': line[index]} + attribute = {"type": self.header[index], "value": line[index]} attribute.update(base_attribute) self.misp_event.add_attribute(**attribute) else: for line in self.data: for index in types_indexes: - self.misp_event.add_attribute(**{'type': self.header[index], 'value': line[index]}) + self.misp_event.add_attribute(**{"type": self.header[index], "value": line[index]}) ################################################################################ # Parsing standard MISP csv format # @@ -157,11 +224,31 @@ def __parse_misp_csv(self): objects = {} attribute_fields = self.header[:1] + self.header[2:8] for line in self.data: - a_uuid, _, category, _type, value, comment, ids, timestamp, relation, tag, o_uuid, name, _ = line[:self.fields_number] - attribute = {t: v.strip('"') for t, v in zip(attribute_fields, (a_uuid, category, _type, value, comment, ids, timestamp))} - attribute['to_ids'] = True if attribute['to_ids'] == '1' else False + ( + a_uuid, + _, + category, + _type, + value, + comment, + ids, + timestamp, + relation, + tag, + o_uuid, + name, + _, + ) = line[: self.fields_number] + attribute = { + t: v.strip('"') + for t, v in zip( + attribute_fields, + (a_uuid, category, _type, value, comment, ids, timestamp), + ) + } + attribute["to_ids"] = True if attribute["to_ids"] == "1" else False if tag: - attribute['Tag'] = [{'name': t.strip()} for t in tag.split(',')] + attribute["Tag"] = [{"name": t.strip()} for t in tag.split(",")] if relation: if o_uuid not in objects: objects[o_uuid] = MISPObject(name) @@ -216,68 +303,79 @@ def __create_standard_attribute(self, line, indexes): return {self.header[index]: line[index] for index in indexes if line[index]} def __check_category(self, attribute): - category = attribute['category'] + category = attribute["category"] if category in self.categories: return if category.capitalize() in self.categories: - attribute['category'] = category.capitalize() + attribute["category"] = category.capitalize() return - del attribute['category'] + del attribute["category"] @staticmethod def __deal_with_ids(attribute): - attribute['to_ids'] = True if attribute['to_ids'] == '1' else False + attribute["to_ids"] = True if attribute["to_ids"] == "1" else False @staticmethod def __deal_with_tags(attribute): - if 'Tag' in attribute.keys(): - attribute['Tag'] = [{'name': tag.strip()} for tag in attribute['Tag'].split(',')] + if "Tag" in attribute.keys(): + attribute["Tag"] = [{"name": tag.strip()} for tag in attribute["Tag"].split(",")] def __get_score(self): - score = 1 if 'to_ids' in self.header else 0 - if 'attribute_tag' in self.header: + score = 1 if "to_ids" in self.header else 0 + if "attribute_tag" in self.header: score += 2 - if 'category' in self.header: + if "category" in self.header: score += 4 return score def __finalize_results(self): event = self.misp_event.to_dict() - self.results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} + self.results = {key: event[key] for key in ("Attribute", "Object") if (key in event and event[key])} def __any_mandatory_misp_field(header): - return any(field in header for field in ('type', 'value')) + return any(field in header for field in ("type", "value")) def __special_parsing(data, delimiter): - return list(tuple(part.strip() for part in line[0].split(delimiter)) for line in csv.reader(io.TextIOWrapper(io.BytesIO(data.encode()), encoding='utf-8')) if line and not line[0].startswith('#')) + return list( + tuple(part.strip() for part in line[0].split(delimiter)) + for line in csv.reader(io.TextIOWrapper(io.BytesIO(data.encode()), encoding="utf-8")) + if line and not line[0].startswith("#") + ) def __standard_parsing(data): - return list(tuple(part.strip() for part in line) for line in csv.reader(io.TextIOWrapper(io.BytesIO(data.encode()), encoding='utf-8')) if line and not line[0].startswith('#')) + return list( + tuple(part.strip() for part in line) + for line in csv.reader(io.TextIOWrapper(io.BytesIO(data.encode()), encoding="utf-8")) + if line and not line[0].startswith("#") + ) def dict_handler(request: dict): - if request.get('data'): + if request.get("data"): try: - data = base64.b64decode(request['data']).decode('utf-8') + data = base64.b64decode(request["data"]).decode("utf-8") except UnicodeDecodeError: - misperrors['error'] = "Input is not valid UTF-8" + misperrors["error"] = "Input is not valid UTF-8" return misperrors else: - misperrors['error'] = "Unsupported attributes type" + misperrors["error"] = "Unsupported attributes type" return misperrors - has_header = request['config'].get('has_header') - has_header = True if has_header == '1' else False - header = request['config']['header'].split(',') if request['config'].get('header').strip() else [] - delimiter = request['config']['special_delimiter'] if request['config'].get('special_delimiter').strip() else ',' - data = __standard_parsing(data) if delimiter == ',' else __special_parsing(data, delimiter) + has_header = request["config"].get("has_header") + has_header = True if has_header == "1" else False + header = request["config"]["header"].split(",") if request["config"].get("header").strip() else [] + delimiter = request["config"]["special_delimiter"] if request["config"].get("special_delimiter").strip() else "," + data = __standard_parsing(data) if delimiter == "," else __special_parsing(data, delimiter) if not header: if has_header: header = data.pop(0) else: - misperrors['error'] = "Configuration error. Provide a header or use the one within the csv file and tick the checkbox 'Has_header'." + misperrors["error"] = ( + "Configuration error. Provide a header or use the one within the csv file and tick the checkbox" + " 'Has_header'." + ) return misperrors else: header = [h.strip() for h in header] @@ -287,27 +385,51 @@ def dict_handler(request: dict): header = misp_standard_csv_header description = MISPEvent().describe_types - misp_types = description['types'] + misp_types = description["types"] for h in header: - if not any((h in misp_types, h in misp_extended_csv_header, h in ('', ' ', '_', 'object_id'))): - misperrors['error'] = 'Wrong header field: {}. Please use a header value that can be recognized by MISP (or alternatively skip it using a whitespace).'.format(h) + if not any( + ( + h in misp_types, + h in misp_extended_csv_header, + h in ("", " ", "_", "object_id"), + ) + ): + misperrors["error"] = ( + "Wrong header field: {}. Please use a header value that can be recognized by MISP (or alternatively" + " skip it using a whitespace).".format(h) + ) return misperrors - from_misp = all((h in misp_extended_csv_header or h in ('', ' ', '_', 'object_id') for h in header)) + from_misp = all((h in misp_extended_csv_header or h in ("", " ", "_", "object_id") for h in header)) if from_misp: if not __any_mandatory_misp_field(header): - misperrors['error'] = 'Please make sure the data you try to import can be identified with a type/value combinaison.' + misperrors["error"] = ( + "Please make sure the data you try to import can be identified with a type/value combinaison." + ) return misperrors else: if __any_mandatory_misp_field(header): - wrong_types = tuple(wrong_type for wrong_type in ('type', 'value') if wrong_type in header) - misperrors['error'] = 'Error with the following header: {}. It contains the following field(s): {}, which is(are) already provided by the usage of at least on MISP attribute type in the header.'.format(header, 'and'.join(wrong_types)) + wrong_types = tuple(wrong_type for wrong_type in ("type", "value") if wrong_type in header) + misperrors["error"] = ( + "Error with the following header: {}. It contains the following field(s): {}, which is(are) already" + " provided by the usage of at least on MISP attribute type in the header.".format( + header, "and".join(wrong_types) + ) + ) return misperrors - csv_parser = CsvParser(header, has_header, delimiter, data, from_misp, misp_types, description['categories']) + csv_parser = CsvParser( + header, + has_header, + delimiter, + data, + from_misp, + misp_types, + description["categories"], + ) # build the attributes result = csv_parser.parse_csv() - if 'error' in result: + if "error" in result: return result - return {'results': csv_parser.results} + return {"results": csv_parser.results} def introspection(): @@ -315,5 +437,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/import_mod/cuckooimport.py b/misp_modules/modules/import_mod/cuckooimport.py index 626f1cbcf..17172fd1e 100755 --- a/misp_modules/modules/import_mod/cuckooimport.py +++ b/misp_modules/modules/import_mod/cuckooimport.py @@ -1,39 +1,46 @@ -import json import base64 import io +import json import logging import posixpath import stat import tarfile import zipfile -from pymisp import MISPEvent, MISPObject, MISPAttribute -from pymisp.tools import make_binary_objects from collections import OrderedDict +from pymisp import MISPAttribute, MISPEvent, MISPObject +from pymisp.tools import make_binary_objects + log = logging.getLogger(__name__) -misperrors = {'error': 'Error'} +misperrors = {"error": "Error"} moduleinfo = { - 'version': '1.1', - 'author': 'Pierre-Jean Grenier', - 'module-type': ['import'], - 'name': 'Cuckoo Sandbox Import', - 'description': 'Module to import Cuckoo JSON.', - 'logo': 'cuckoo.png', - 'requirements': [], - 'features': 'Import a Cuckoo archive (zipfile or bzip2 tarball), either downloaded manually or exported from the API (/tasks/report//all).', - 'references': ['https://cuckoosandbox.org/', 'https://github.com/cuckoosandbox/cuckoo'], - 'input': 'Cuckoo JSON file', - 'output': 'MISP Event attributes', + "version": "1.1", + "author": "Pierre-Jean Grenier", + "module-type": ["import"], + "name": "Cuckoo Sandbox Import", + "description": "Module to import Cuckoo JSON.", + "logo": "cuckoo.png", + "requirements": [], + "features": ( + "Import a Cuckoo archive (zipfile or bzip2 tarball), either downloaded manually or exported from the API" + " (/tasks/report//all)." + ), + "references": [ + "https://cuckoosandbox.org/", + "https://github.com/cuckoosandbox/cuckoo", + ], + "input": "Cuckoo JSON file", + "output": "MISP Event attributes", } moduleconfig = [] mispattributes = { - 'inputSource': ['file'], - 'output': ['MISP objects', 'malware-sample'], - 'format': 'misp_standard', + "inputSource": ["file"], + "output": ["MISP objects", "malware-sample"], + "format": "misp_standard", } # Attributes for which we can set the "Artifacts dropped" @@ -58,6 +65,7 @@ class PrettyDict(OrderedDict): This class is just intended for a pretty print of its keys and values. """ + MAX_SIZE = 30 def __str__(self): @@ -65,9 +73,9 @@ def __str__(self): for k, v in self.items(): v = str(v) if len(v) > self.MAX_SIZE: - k += ',cut' - v = v[:self.MAX_SIZE] - v.replace('\n', ' ') + k += ",cut" + v = v[: self.MAX_SIZE] + v.replace("\n", " ") tmp.append((k, v)) return "; ".join(f"({k}) {v}" for k, v in tmp) @@ -82,12 +90,11 @@ def search_objects(event, name, attributes=[]): match = filter( lambda obj: all( obj.name == name - and (obj_relation, str(attr_value)) in map( - lambda attr: (attr.object_relation, str(attr.value)), - obj.attributes - ) + and (obj_relation, str(attr_value)) + in map(lambda attr: (attr.object_relation, str(attr.value)), obj.attributes) for obj_relation, attr_value in attributes - ), event.objects + ), + event.objects, ) return match @@ -98,15 +105,11 @@ def find_process_by_pid(event, pid): only return the first one. @ param pid: integer or str """ - generator = search_objects( - event, - "process", - (('pid', pid),) - ) + generator = search_objects(event, "process", (("pid", pid),)) return next(generator, None) -class CuckooParser(): +class CuckooParser: # This dict is used to generate the userConfig and link the different # options to the corresponding method of the parser. This way, we avoid # redundancy and make future changes easier (instead of for instance @@ -121,89 +124,89 @@ class CuckooParser(): "Sandbox info": { "method": lambda self: self.add_sandbox_info(), "userConfig": { - 'type': 'Boolean', - 'message': "Add info related to the sandbox", - 'checked': 'true', + "type": "Boolean", + "message": "Add info related to the sandbox", + "checked": "true", }, }, "Upload sample": { "method": lambda self: self.add_sample(), "userConfig": { - 'type': 'Boolean', - 'message': "Upload the sample", - 'checked': 'true', + "type": "Boolean", + "message": "Upload the sample", + "checked": "true", }, }, "Processes": { "method": lambda self: self.add_process_tree(), "userConfig": { - 'type': 'Boolean', - 'message': "Add info related to the processes", - 'checked': 'true', + "type": "Boolean", + "message": "Add info related to the processes", + "checked": "true", }, }, "DNS": { "method": lambda self: self.add_dns(), "userConfig": { - 'type': 'Boolean', - 'message': "Add DNS queries/answers", - 'checked': 'true', + "type": "Boolean", + "message": "Add DNS queries/answers", + "checked": "true", }, }, "TCP": { "method": lambda self: self.add_network("tcp"), "userConfig": { - 'type': 'Boolean', - 'message': "Add TCP connections", - 'checked': 'true', + "type": "Boolean", + "message": "Add TCP connections", + "checked": "true", }, }, "UDP": { "method": lambda self: self.add_network("udp"), "userConfig": { - 'type': 'Boolean', - 'message': "Add UDP connections", - 'checked': 'true', + "type": "Boolean", + "message": "Add UDP connections", + "checked": "true", }, }, "HTTP": { "method": lambda self: self.add_http(), "userConfig": { - 'type': 'Boolean', - 'message': "Add HTTP requests", - 'checked': 'true', + "type": "Boolean", + "message": "Add HTTP requests", + "checked": "true", }, }, "Signatures": { "method": lambda self: self.add_signatures(), "userConfig": { - 'type': 'Boolean', - 'message': "Add Cuckoo's triggered signatures", - 'checked': 'true', + "type": "Boolean", + "message": "Add Cuckoo's triggered signatures", + "checked": "true", }, }, "Screenshots": { "method": lambda self: self.add_screenshots(), "userConfig": { - 'type': 'Boolean', - 'message': "Upload the screenshots", - 'checked': 'true', + "type": "Boolean", + "message": "Upload the screenshots", + "checked": "true", }, }, "Dropped files": { "method": lambda self: self.add_dropped_files(), "userConfig": { - 'type': 'Boolean', - 'message': "Upload the dropped files", - 'checked': 'true', + "type": "Boolean", + "message": "Upload the dropped files", + "checked": "true", }, }, "Dropped buffers": { "method": lambda self: self.add_dropped_buffers(), "userConfig": { - 'type': 'Boolean', - 'message': "Upload the dropped buffers", - 'checked': 'true', + "type": "Boolean", + "message": "Upload the dropped buffers", + "checked": "true", }, }, } @@ -216,10 +219,7 @@ def __init__(self, config): self.config = { # if an option is missing (we receive None as a value), # fall back to the default specified in the options - key: int( - on if on is not None - else self.options[key]["userConfig"]["checked"] == 'true' - ) + key: int(on if on is not None else self.options[key]["userConfig"]["checked"] == "true") for key, on in config.items() } @@ -227,11 +227,10 @@ def get_file(self, relative_filepath): """Return an io.BufferedIOBase for the corresponding relative_filepath in the Cuckoo archive. If not found, return an empty io.BufferedReader to avoid fatal errors.""" - blackhole = io.BufferedReader(open('/dev/null', 'rb')) + blackhole = io.BufferedReader(open("/dev/null", "rb")) res = self.files.get(relative_filepath, blackhole) if res == blackhole: - log.debug(f"Did not find file {relative_filepath}, " - f"returned an empty file instead") + log.debug(f"Did not find file {relative_filepath}, returned an empty file instead") return res def read_archive(self, archive_encoded): @@ -243,19 +242,18 @@ def read_archive(self, archive_encoded): if zipfile.is_zipfile(buf_io): # the archive was probably downloaded from the WebUI buf_io.seek(0) # don't forget this not to read an empty buffer - z = zipfile.ZipFile(buf_io, 'r') + z = zipfile.ZipFile(buf_io, "r") self.files = { info.filename: z.open(info) for info in z.filelist # only extract the regular files and dirs, we don't # want any symbolic link - if stat.S_ISREG(info.external_attr >> 16) - or stat.S_ISDIR(info.external_attr >> 16) + if stat.S_ISREG(info.external_attr >> 16) or stat.S_ISDIR(info.external_attr >> 16) } else: # the archive was probably downloaded from the API buf_io.seek(0) # don't forget this not to read an empty buffer - f = tarfile.open(fileobj=buf_io, mode='r:bz2') + f = tarfile.open(fileobj=buf_io, mode="r:bz2") self.files = { info.name: f.extractfile(info) for info in f.getmembers() @@ -289,19 +287,20 @@ def read_malware(self): def add_sandbox_info(self): info = self.report.get("info", {}) if not info: - log.warning("The 'info' field was not found " - "in the report, skipping") + log.warning("The 'info' field was not found in the report, skipping") return False - o = MISPObject(name='sandbox-report') - o.add_attribute('score', info['score']) - o.add_attribute('sandbox-type', 'on-premise') - o.add_attribute('on-premise-sandbox', 'cuckoo') - o.add_attribute('raw-report', - f'started on:{info["machine"]["started_on"]} ' - f'duration:{info["duration"]}s ' - f'vm:{info["machine"]["name"]}/' - f'{info["machine"]["label"]}') + o = MISPObject(name="sandbox-report") + o.add_attribute("score", info["score"]) + o.add_attribute("sandbox-type", "on-premise") + o.add_attribute("on-premise-sandbox", "cuckoo") + o.add_attribute( + "raw-report", + f'started on:{info["machine"]["started_on"]} ' + f'duration:{info["duration"]}s ' + f'vm:{info["machine"]["name"]}/' + f'{info["machine"]["label"]}', + ) self.event.add_object(o) def add_sample(self): @@ -309,8 +308,7 @@ def add_sample(self): target = self.report.get("target", {}) category = target.get("category", "") if not category: - log.warning("Could not find info about the sample " - "in the report, skipping") + log.warning("Could not find info about the sample in the report, skipping") return False if category == "file": @@ -323,7 +321,14 @@ def add_sample(self): file_o.comment = "Submitted sample" # fix categories - for obj in filter(None, (file_o, bin_type_o, *bin_section_li,)): + for obj in filter( + None, + ( + file_o, + bin_type_o, + *bin_section_li, + ), + ): for attr in obj.attributes: if attr.type in PAYLOAD_DELIVERY: attr.category = "Payload delivery" @@ -331,9 +336,9 @@ def add_sample(self): elif category == "url": log.debug("Sample is a URL") - o = MISPObject(name='url') - o.add_attribute('url', target['url']) - o.add_attribute('text', "Submitted URL") + o = MISPObject(name="url") + o.add_attribute("url", target["url"]) + o.add_attribute("text", "Submitted URL") self.event.add_object(o) def add_http(self): @@ -345,13 +350,12 @@ def add_http(self): return False for request in http: - o = MISPObject(name='http-request') - o.add_attribute('host', request['host']) - o.add_attribute('method', request['method']) - o.add_attribute('uri', request['uri']) - o.add_attribute('user-agent', request['user-agent']) - o.add_attribute('text', f"count:{request['count']} " - f"port:{request['port']}") + o = MISPObject(name="http-request") + o.add_attribute("host", request["host"]) + o.add_attribute("method", request["method"]) + o.add_attribute("uri", request["uri"]) + o.add_attribute("user-agent", request["user-agent"]) + o.add_attribute("text", f"count:{request['count']} port:{request['port']}") self.event.add_object(o) def add_network(self, proto=None): @@ -369,23 +373,23 @@ def add_network(self, proto=None): # sort by time to get the "first packet seen" right li_conn.sort(key=lambda x: x["time"]) for conn in li_conn: - src = conn['src'] - dst = conn['dst'] - sport = conn['sport'] - dport = conn['dport'] + src = conn["src"] + dst = conn["dst"] + sport = conn["sport"] + dport = conn["dport"] if (src, sport, dst, dport) in from_to: continue from_to.append((src, sport, dst, dport)) - o = MISPObject(name='network-connection') - o.add_attribute('ip-src', src) - o.add_attribute('ip-dst', dst) - o.add_attribute('src-port', sport) - o.add_attribute('dst-port', dport) - o.add_attribute('layer3-protocol', "IP") - o.add_attribute('layer4-protocol', proto.upper()) - o.add_attribute('first-packet-seen', conn['time']) + o = MISPObject(name="network-connection") + o.add_attribute("ip-src", src) + o.add_attribute("ip-dst", dst) + o.add_attribute("src-port", sport) + o.add_attribute("dst-port", dport) + o.add_attribute("layer3-protocol", "IP") + o.add_attribute("layer4-protocol", proto.upper()) + o.add_attribute("first-packet-seen", conn["time"]) self.event.add_object(o) def add_dns(self): @@ -397,12 +401,12 @@ def add_dns(self): return False for record in dns: - o = MISPObject(name='dns-record') - o.add_attribute('text', f"request type:{record['type']}") - o.add_attribute('queried-domain', record['request']) + o = MISPObject(name="dns-record") + o.add_attribute("text", f"request type:{record['type']}") + o.add_attribute("queried-domain", record["request"]) for answer in record.get("answers", []): if answer["type"] in ("A", "AAAA"): - o.add_attribute('a-record', answer['data']) + o.add_attribute("a-record", answer["data"]) # TODO implement MX/NS self.event.add_object(o) @@ -416,7 +420,7 @@ def _get_marks_str(self, marks): marks_strings.append(str(m)) elif m_type == "ioc": - marks_strings.append(m['ioc']) + marks_strings.append(m["ioc"]) elif m_type == "call": call = m["call"] @@ -444,8 +448,7 @@ def _add_ttp(self, attribute, ttp_short, ttp_num): - ttp_num: formatted as "T"+int (eg. T1003) """ - attribute.add_tag(f'misp-galaxy:mitre-attack-pattern=' - f'"{ttp_short} - {ttp_num}"') + attribute.add_tag(f"misp-galaxy:mitre-attack-pattern=" f'"{ttp_short} - {ttp_num}"') def add_signatures(self): """Add the Cuckoo signatures, with as many details as possible @@ -455,13 +458,13 @@ def add_signatures(self): log.info("No signature found in the report") return False - o = MISPObject(name='sb-signature') - o.add_attribute('software', "Cuckoo") + o = MISPObject(name="sb-signature") + o.add_attribute("software", "Cuckoo") for sign in signatures: marks = sign["marks"] marks_strings = self._get_marks_str(marks) - summary = sign['description'] + summary = sign["description"] if marks_strings: summary += "\n---\n" @@ -469,17 +472,14 @@ def add_signatures(self): description = summary + "\n".join(marks_strings) a = MISPAttribute() - a.from_dict(type='text', value=description) + a.from_dict(type="text", value=description) for ttp_num, desc in sign.get("ttp", {}).items(): ttp_short = desc["short"] self._add_ttp(a, ttp_short, ttp_num) # this signature was triggered by the processes with the following # PIDs, we can create references - triggered_by_pids = filter( - None, - (m.get("pid", None) for m in marks) - ) + triggered_by_pids = filter(None, (m.get("pid", None) for m in marks)) # remove redundancy triggered_by_pids = set(triggered_by_pids) for pid in triggered_by_pids: @@ -487,7 +487,7 @@ def add_signatures(self): if process_o: process_o.add_reference(a, "triggers") - o.add_attribute('signature', **a) + o.add_attribute("signature", **a) self.event.add_object(o) @@ -498,18 +498,18 @@ def _handle_process(self, proc, accu): List the objects to be added, based on the tree, into the `accu` list. The `accu` list uses a DFS-like order. """ - o = MISPObject(name='process') + o = MISPObject(name="process") accu.append(o) - o.add_attribute('pid', proc['pid']) - o.add_attribute('command-line', proc['command_line']) - o.add_attribute('name', proc['process_name']) - o.add_attribute('parent-pid', proc['ppid']) - for child in proc.get('children', []): + o.add_attribute("pid", proc["pid"]) + o.add_attribute("command-line", proc["command_line"]) + o.add_attribute("name", proc["process_name"]) + o.add_attribute("parent-pid", proc["ppid"]) + for child in proc.get("children", []): pos_child = len(accu) - o.add_attribute('child-pid', child['pid']) + o.add_attribute("child-pid", child["pid"]) self._handle_process(child, accu) child_obj = accu[pos_child] - child_obj.add_reference(o, 'child-of') + child_obj.add_reference(o, "child-of") return o @@ -549,26 +549,28 @@ def get_relpath(self, path): def add_screenshots(self): """Add the screenshots taken by Cuckoo in a sandbox-report object""" - screenshots = self.report.get('screenshots', []) + screenshots = self.report.get("screenshots", []) if not screenshots: log.info("No screenshot found in the report, skipping") return False - o = MISPObject(name='sandbox-report') - o.add_attribute('sandbox-type', 'on-premise') - o.add_attribute('on-premise-sandbox', "cuckoo") + o = MISPObject(name="sandbox-report") + o.add_attribute("sandbox-type", "on-premise") + o.add_attribute("on-premise-sandbox", "cuckoo") for shot in screenshots: # The path given by Cuckoo is an absolute path, but we need a path # relative to the analysis folder. - path = self.get_relpath(shot['path']) + path = self.get_relpath(shot["path"]) img = self.get_file(path) # .decode('utf-8') in order to avoid the b'' format - img_data = base64.b64encode(img.read()).decode('utf-8') + img_data = base64.b64encode(img.read()).decode("utf-8") filename = posixpath.basename(path) o.add_attribute( - "sandbox-file", value=filename, - data=img_data, type='attachment', + "sandbox-file", + value=filename, + data=img_data, + type="attachment", category="External analysis", ) @@ -588,13 +590,21 @@ def _get_dropped_objs(self, path, filename=None, comment=None): dropped_binary = io.BytesIO(dropped_file.read()) # create ad hoc objects file_o, bin_type_o, bin_section_li = make_binary_objects( - pseudofile=dropped_binary, filename=filename, + pseudofile=dropped_binary, + filename=filename, ) if comment: file_o.comment = comment # fix categories - for obj in filter(None, (file_o, bin_type_o, *bin_section_li,)): + for obj in filter( + None, + ( + file_o, + bin_type_o, + *bin_section_li, + ), + ): for attr in obj.attributes: if attr.type in ARTIFACTS_DROPPED: attr.category = "Artifacts dropped" @@ -609,7 +619,7 @@ def _add_yara(self, obj, yara_dict): obj.add_attribute( "text", f"Yara match\n(name) {name}\n(description) {description}", - comment="Yara match" + comment="Yara match", ) def add_dropped_files(self): @@ -632,47 +642,41 @@ def add_dropped_files(self): original_path = d.get("filepath", "") sha256 = d.get("sha256", "") if original_path and sha256: - log.debug(f"Will now try to restore original filename from " - f"path {original_path}") + log.debug(f"Will now try to restore original filename from path {original_path}") try: s = filename.split("_") if not s: - raise Exception("unexpected filename read " - "in the report") + raise Exception("unexpected filename read in the report") sha256_first_8_bytes = s[0] original_name = s[1] # check our assumptions are valid, if so we can safely # restore the filename, if not the format may have changed # so we'll keep the filename of the report - if sha256.startswith(sha256_first_8_bytes) and \ - original_path.lower().endswith(original_name) and \ - filename not in original_path.lower(): + if ( + sha256.startswith(sha256_first_8_bytes) + and original_path.lower().endswith(original_name) + and filename not in original_path.lower() + ): # we can restore the original case of the filename position = original_path.lower().rindex(original_name) filename = original_path[position:] - log.debug(f"Successfully restored original filename: " - f"{filename}") + log.debug(f"Successfully restored original filename: {filename}") else: - raise Exception("our assumptions were wrong, " - "filename format may have changed") + raise Exception("our assumptions were wrong, filename format may have changed") except Exception as e: log.debug(f"Cannot restore filename: {e}") if not filename: filename = "NO NAME FOUND IN THE REPORT" - log.warning(f'No filename found for dropped file! ' - f'Will use "{filename}"') + log.warning(f"No filename found for dropped file! " f'Will use "{filename}"') file_o, bin_type_o, bin_section_o = self._get_dropped_objs( - self.get_relpath(d['path']), - filename=filename, - comment="Dropped file" + self.get_relpath(d["path"]), filename=filename, comment="Dropped file" ) self._add_yara(file_o, d.get("yara", [])) - file_o.add_attribute("fullpath", original_path, - category="Artifacts dropped") + file_o.add_attribute("fullpath", original_path, category="Artifacts dropped") # why is this a list? for when various programs drop the same file? for pid in d.get("pids", []): @@ -686,7 +690,7 @@ def add_dropped_files(self): self.event.add_object(file_o) def add_dropped_buffers(self): - """"Upload the dropped buffers as file objects""" + """ "Upload the dropped buffers as file objects""" buffer = self.report.get("buffer", []) if not buffer: log.info("No dropped buffer found, skipping") @@ -694,9 +698,9 @@ def add_dropped_buffers(self): for i, buf in enumerate(buffer): file_o, bin_type_o, bin_section_o = self._get_dropped_objs( - self.get_relpath(buf['path']), + self.get_relpath(buf["path"]), filename=f"buffer {i}", - comment="Dropped buffer" + comment="Dropped buffer", ) self._add_yara(file_o, buf.get("yara", [])) self.event.add_object(file_o) @@ -719,31 +723,24 @@ def handler(q=False): return False q = json.loads(q) - data = q['data'] + data = q["data"] - parser = CuckooParser(q['config']) + parser = CuckooParser(q["config"]) parser.read_archive(data) parser.parse() event = parser.get_misp_event() event = json.loads(event.to_json()) - results = { - key: event[key] - for key in ('Attribute', 'Object') - if (key in event and event[key]) - } - return {'results': results} + results = {key: event[key] for key in ("Attribute", "Object") if (key in event and event[key])} + return {"results": results} def introspection(): - userConfig = { - key: o["userConfig"] - for key, o in CuckooParser.options.items() - } - mispattributes['userConfig'] = userConfig + userConfig = {key: o["userConfig"] for key, o in CuckooParser.options.items()} + mispattributes["userConfig"] = userConfig return mispattributes def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/import_mod/email_import.py b/misp_modules/modules/import_mod/email_import.py index 411185e6b..0962c8542 100644 --- a/misp_modules/modules/import_mod/email_import.py +++ b/misp_modules/modules/import_mod/email_import.py @@ -1,39 +1,44 @@ #!/usr/bin/env python3 -import json import base64 -import zipfile +import json import re +import zipfile from html.parser import HTMLParser -from pymisp.tools import EMailObject, make_binary_objects, URLObject from io import BytesIO from pathlib import Path +from pymisp.tools import EMailObject, URLObject, make_binary_objects -misperrors = {'error': 'Error'} +misperrors = {"error": "Error"} -mispattributes = {'inputSource': ['file'], 'output': ['MISP objects'], - 'format': 'misp_standard'} +mispattributes = { + "inputSource": ["file"], + "output": ["MISP objects"], + "format": "misp_standard", +} moduleinfo = { - 'version': '0.2', - 'author': 'Seamus Tuohy, Raphaël Vinot', - 'description': 'Email import module for MISP', - 'module-type': ['import'], - 'name': 'Email Import', - 'requirements': [], - 'features': 'This module can be used to import e-mail text as well as attachments and urls.\n3 configuration parameters are then used to unzip attachments, guess zip attachment passwords, and extract urls: set each one of them to True or False to process or not the respective corresponding actions.', - 'references': [], - 'input': 'E-mail file', - 'output': 'MISP Event attributes', - 'logo': '', + "version": "0.2", + "author": "Seamus Tuohy, Raphaël Vinot", + "description": "Email import module for MISP", + "module-type": ["import"], + "name": "Email Import", + "requirements": [], + "features": ( + "This module can be used to import e-mail text as well as attachments and urls.\n3 configuration parameters are" + " then used to unzip attachments, guess zip attachment passwords, and extract urls: set each one of them to" + " True or False to process or not the respective corresponding actions." + ), + "references": [], + "input": "E-mail file", + "output": "MISP Event attributes", + "logo": "", } # unzip_attachments : Unzip all zip files that are not password protected # guess_zip_attachment_passwords : This attempts to unzip all password protected zip files using all the strings found in the email body and subject # extract_urls : This attempts to extract all URL's from text/html parts of the email -moduleconfig = ["unzip_attachments", - "guess_zip_attachment_passwords", - "extract_urls"] +moduleconfig = ["unzip_attachments", "guess_zip_attachment_passwords", "extract_urls"] def dict_handler(request: dict): @@ -45,7 +50,7 @@ def dict_handler(request: dict): # Check if we were given a configuration config = request.get("config", {}) # Don't be picky about how the user chooses to say yes to these - acceptable_config_yes = ['y', 'yes', 'true', 't'] + acceptable_config_yes = ["y", "yes", "true", "t"] # Do we unzip attachments we find? unzip = config.get("unzip_attachments", None) @@ -69,13 +74,42 @@ def dict_handler(request: dict): for attachment_name, attachment in email_object.attachments: # Create file objects for the attachments if not attachment_name: - attachment_name = 'NameMissing.txt' + attachment_name = "NameMissing.txt" temp_filename = Path(attachment_name) - zipped_files = ["doc", "docx", "dot", "dotx", "xls", "xlsx", "xlm", "xla", - "xlc", "xlt", "xltx", "xlw", "ppt", "pptx", "pps", "ppsx", - "pot", "potx", "potx", "sldx", "odt", "ods", "odp", "odg", - "odf", "fodt", "fods", "fodp", "fodg", "ott", "uot"] + zipped_files = [ + "doc", + "docx", + "dot", + "dotx", + "xls", + "xlsx", + "xlm", + "xla", + "xlc", + "xlt", + "xltx", + "xlw", + "ppt", + "pptx", + "pps", + "ppsx", + "pot", + "potx", + "potx", + "sldx", + "odt", + "ods", + "odp", + "odg", + "odf", + "fodt", + "fods", + "fodp", + "fodg", + "ott", + "uot", + ] # Attempt to unzip the attachment and return its files if unzip and temp_filename.suffix[1:] not in zipped_files: try: @@ -84,49 +118,66 @@ def dict_handler(request: dict): if zip_pass_crack is True: password = test_zip_passwords(attachment, password_list) if password: - unzip_attachment(attachment_name, attachment, email_object, file_objects, password) + unzip_attachment( + attachment_name, + attachment, + email_object, + file_objects, + password, + ) else: # Inform the analyst that we could not crack password - f_object, main_object, sections = make_binary_objects(pseudofile=attachment, filename=attachment_name, standalone=False) + f_object, main_object, sections = make_binary_objects( + pseudofile=attachment, + filename=attachment_name, + standalone=False, + ) f_object.comment = "Encrypted Zip: Password could not be cracked from message" file_objects.append(f_object) file_objects.append(main_object) file_objects += sections - email_object.add_reference(f_object.uuid, 'includes', 'Email attachment') + email_object.add_reference(f_object.uuid, "includes", "Email attachment") except zipfile.BadZipFile: # Attachment is not a zipfile # Just straight add the file - f_object, main_object, sections = make_binary_objects(pseudofile=attachment, filename=attachment_name, standalone=False) + f_object, main_object, sections = make_binary_objects( + pseudofile=attachment, filename=attachment_name, standalone=False + ) file_objects.append(f_object) file_objects.append(main_object) file_objects += sections - email_object.add_reference(f_object.uuid, 'includes', 'Email attachment') + email_object.add_reference(f_object.uuid, "includes", "Email attachment") else: # Just straight add the file - f_object, main_object, sections = make_binary_objects(pseudofile=attachment, filename=attachment_name, standalone=False) + f_object, main_object, sections = make_binary_objects( + pseudofile=attachment, filename=attachment_name, standalone=False + ) file_objects.append(f_object) file_objects.append(main_object) file_objects += sections - email_object.add_reference(f_object.uuid, 'includes', 'Email attachment') + email_object.add_reference(f_object.uuid, "includes", "Email attachment") - mail_body = email_object.email.get_body(preferencelist=('html', 'plain')) + mail_body = email_object.email.get_body(preferencelist=("html", "plain")) if extract_urls and mail_body: - charset = mail_body.get_content_charset('utf-8') - if mail_body.get_content_type() == 'text/html': + charset = mail_body.get_content_charset("utf-8") + if mail_body.get_content_type() == "text/html": url_parser = HTMLURLParser() - url_parser.feed(mail_body.get_payload(decode=True).decode(charset, errors='ignore')) + url_parser.feed(mail_body.get_payload(decode=True).decode(charset, errors="ignore")) urls = url_parser.urls else: - urls = re.findall(r'https?://(?:[-\w.]|(?:%[\da-fA-F]{2}))+', mail_body.get_payload(decode=True).decode(charset, errors='ignore')) + urls = re.findall( + r"https?://(?:[-\w.]|(?:%[\da-fA-F]{2}))+", + mail_body.get_payload(decode=True).decode(charset, errors="ignore"), + ) for url in urls: if not url: continue url_object = URLObject(url, standalone=False) file_objects.append(url_object) - email_object.add_reference(url_object.uuid, 'includes', 'URL in email body') + email_object.add_reference(url_object.uuid, "includes", "URL in email body") objects = [email_object.to_dict()] if file_objects: objects += [o.to_dict() for o in file_objects if o] - r = {'results': {'Object': objects}} + r = {"results": {"Object": objects}} return r @@ -149,18 +200,18 @@ def unzip_attachment(filename, data, email_object, file_objects, password=None): comment = f'Extracted from {filename} with password "{password}"' password = str.encode(password) # Byte encoded password required else: - comment = f'Extracted from {filename}' + comment = f"Extracted from {filename}" for zip_file_name in zf.namelist(): # Get all files in the zip file - with zf.open(zip_file_name, mode='r', pwd=password) as fp: + with zf.open(zip_file_name, mode="r", pwd=password) as fp: file_data = BytesIO(fp.read()) - f_object, main_object, sections = make_binary_objects(pseudofile=file_data, - filename=zip_file_name, - standalone=False) + f_object, main_object, sections = make_binary_objects( + pseudofile=file_data, filename=zip_file_name, standalone=False + ) f_object.comment = comment file_objects.append(f_object) file_objects.append(main_object) file_objects += sections - email_object.add_reference(f_object.uuid, 'includes', 'Email attachment') + email_object.add_reference(f_object.uuid, "includes", "Email attachment") def test_zip_passwords(data, test_passwords): @@ -187,7 +238,7 @@ def test_zip_passwords(data, test_passwords): def get_zip_passwords(message): - """ Parse message for possible zip password combinations. + """Parse message for possible zip password combinations. Args: message (email.message) Email message object to parse. @@ -197,12 +248,32 @@ def get_zip_passwords(message): malware_passwords = ["infected", "malware"] possible_passwords += malware_passwords # Commonly used passwords - common_passwords = ["123456", "password", "12345678", "qwerty", - "abc123", "123456789", "111111", "1234567", - "iloveyou", "adobe123", "123123", "sunshine", - "1234567890", "letmein", "1234", "monkey", - "shadow", "sunshine", "12345", "password1", - "princess", "azerty", "trustno1", "000000"] + common_passwords = [ + "123456", + "password", + "12345678", + "qwerty", + "abc123", + "123456789", + "111111", + "1234567", + "iloveyou", + "adobe123", + "123123", + "sunshine", + "1234567890", + "letmein", + "1234", + "monkey", + "shadow", + "sunshine", + "12345", + "password1", + "princess", + "azerty", + "trustno1", + "000000", + ] possible_passwords += common_passwords @@ -213,24 +284,24 @@ def get_zip_passwords(message): charset = part.get_content_charset() if not charset: charset = "utf-8" - if part.get_content_type() == 'text/plain': - body.append(part.get_payload(decode=True).decode(charset, errors='ignore')) - elif part.get_content_type() == 'text/html': + if part.get_content_type() == "text/plain": + body.append(part.get_payload(decode=True).decode(charset, errors="ignore")) + elif part.get_content_type() == "text/html": html_parser = HTMLTextParser() payload = part.get_payload(decode=True) if payload: - html_parser.feed(payload.decode(charset, errors='ignore')) + html_parser.feed(payload.decode(charset, errors="ignore")) for text in html_parser.text_data: body.append(text) raw_text = "\n".join(body).strip() # Add subject to text corpus to parse if "Subject" in message: - subject = " " + message.get('Subject') + subject = " " + message.get("Subject") raw_text += subject # Grab any strings that are marked off by special chars - marking_chars = [["\'", "\'"], ['"', '"'], ['[', ']'], ['(', ')']] + marking_chars = [["'", "'"], ['"', '"'], ["[", "]"], ["(", ")"]] for char_set in marking_chars: regex = re.compile(r"""\{0}([^\{1}]*)\{1}""".format(char_set[0], char_set[1])) marked_off = re.findall(regex, raw_text) @@ -240,7 +311,7 @@ def get_zip_passwords(message): individual_words = re.split(r"\s", raw_text) # Also get words with basic punctuation stripped out # just in case someone places a password in a proper sentence - stripped_words = [i.strip('.,;:?!') for i in individual_words] + stripped_words = [i.strip(".,;:?!") for i in individual_words] unique_words = list(set(individual_words + stripped_words)) possible_passwords += unique_words @@ -248,7 +319,8 @@ def get_zip_passwords(message): class HTMLTextParser(HTMLParser): - """ Parse all text and data from HTML strings.""" + """Parse all text and data from HTML strings.""" + def __init__(self, text_data=None): HTMLParser.__init__(self) if text_data is None: @@ -261,7 +333,8 @@ def handle_data(self, data): class HTMLURLParser(HTMLParser): - """ Parse all href targets from HTML strings.""" + """Parse all href targets from HTML strings.""" + def __init__(self, urls=None): HTMLParser.__init__(self) if urls is None: @@ -270,10 +343,10 @@ def __init__(self, urls=None): self.urls = urls def handle_starttag(self, tag, attrs): - if tag == 'a': - self.urls.append(dict(attrs).get('href')) - if tag == 'img': - self.urls.append(dict(attrs).get('src')) + if tag == "a": + self.urls.append(dict(attrs).get("href")) + if tag == "img": + self.urls.append(dict(attrs).get("src")) def introspection(): @@ -281,10 +354,10 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo -if __name__ == '__main__': - with open('tests/test_no_attach.eml', 'r') as email_file: +if __name__ == "__main__": + with open("tests/test_no_attach.eml", "r") as email_file: dict_handler(json.loads(email_file.read())) diff --git a/misp_modules/modules/import_mod/goamlimport.py b/misp_modules/modules/import_mod/goamlimport.py index afb026684..af526ed1a 100644 --- a/misp_modules/modules/import_mod/goamlimport.py +++ b/misp_modules/modules/import_mod/goamlimport.py @@ -1,92 +1,249 @@ +import base64 import json import time -import base64 import xml.etree.ElementTree as ET + from pymisp import MISPEvent, MISPObject -misperrors = {'error': 'Error'} +misperrors = {"error": "Error"} moduleinfo = { - 'version': 1, - 'author': 'Christian Studer', - 'description': 'Module to import MISP objects about financial transactions from GoAML files.', - 'module-type': ['import'], - 'name': 'GoAML Import', - 'logo': 'goAML.jpg', - 'requirements': ['PyMISP'], - 'features': 'Unlike the GoAML export module, there is here no special feature to import data from GoAML external files, since the module will import MISP Objects with their References on its own, as it is required for the export module to rebuild a valid GoAML document.', - 'references': 'http://goaml.unodc.org/', - 'input': 'GoAML format file, describing financial transactions, with their origin and target (bank accounts, persons or entities).', - 'output': 'MISP objects (transaction, bank-account, person, legal-entity, geolocation), with references, describing financial transactions and their origin and target.', + "version": 1, + "author": "Christian Studer", + "description": "Module to import MISP objects about financial transactions from GoAML files.", + "module-type": ["import"], + "name": "GoAML Import", + "logo": "goAML.jpg", + "requirements": ["PyMISP"], + "features": ( + "Unlike the GoAML export module, there is here no special feature to import data from GoAML external files," + " since the module will import MISP Objects with their References on its own, as it is required for the export" + " module to rebuild a valid GoAML document." + ), + "references": "http://goaml.unodc.org/", + "input": ( + "GoAML format file, describing financial transactions, with their origin and target (bank accounts, persons or" + " entities)." + ), + "output": ( + "MISP objects (transaction, bank-account, person, legal-entity, geolocation), with references, describing" + " financial transactions and their origin and target." + ), } moduleconfig = [] -mispattributes = {'inputSource': ['file'], 'output': ['MISP objects'], - 'format': 'misp_standard'} - -t_from_objects = {'nodes': ['from_person', 'from_account', 'from_entity'], - 'leaves': ['from_funds_code', 'from_country']} -t_to_objects = {'nodes': ['to_person', 'to_account', 'to_entity'], - 'leaves': ['to_funds_code', 'to_country']} -t_person_objects = {'nodes': ['addresses'], - 'leaves': ['first_name', 'middle_name', 'last_name', 'gender', 'title', 'mothers_name', 'birthdate', - 'passport_number', 'passport_country', 'id_number', 'birth_place', 'alias', 'nationality1']} -t_account_objects = {'nodes': ['signatory'], - 'leaves': ['institution_name', 'institution_code', 'swift', 'branch', 'non_banking_insitution', - 'account', 'currency_code', 'account_name', 'iban', 'client_number', 'opened', 'closed', - 'personal_account_type', 'balance', 'date_balance', 'status_code', 'beneficiary', - 'beneficiary_comment', 'comments']} -entity_objects = {'nodes': ['addresses'], - 'leaves': ['name', 'commercial_name', 'incorporation_legal_form', 'incorporation_number', 'business', 'phone']} - -goAMLobjects = {'report': {'nodes': ['reporting_person', 'location'], - 'leaves': ['rentity_id', 'submission_code', 'report_code', 'submission_date', 'currency_code_local']}, - 'reporting_person': {'nodes': ['addresses'], 'leaves': ['first_name', 'middle_name', 'last_name', 'title']}, - 'location': {'nodes': [], 'leaves': ['address_type', 'address', 'city', 'zip', 'country_code', 'state']}, - 'transaction': {'nodes': ['t_from', 't_from_my_client', 't_to', 't_to_my_client'], - 'leaves': ['transactionnumber', 'transaction_location', 'date_transaction', - 'transmode_code', 'amount_local']}, - 't_from': t_from_objects, 't_from_my_client': t_from_objects, - 't_to': t_to_objects, 't_to_my_client': t_to_objects, - 'addresses': {'nodes': ['address'], 'leaves': []}, - 'address': {'nodes': [], 'leaves': ['address_type', 'address', 'city', 'zip', 'country_code', 'state']}, - 'from_person': t_person_objects, 'to_person': t_person_objects, 't_person': t_person_objects, - 'from_account': t_account_objects, 'to_account': t_account_objects, - 'signatory': {'nodes': ['t_person'], 'leaves': []}, - 'from_entity': entity_objects, 'to_entity': entity_objects, - } - -t_account_mapping = {'misp_name': 'bank-account', 'institution_name': 'institution-name', 'institution_code': 'institution-code', - 'iban': 'iban', 'swift': 'swift', 'branch': 'branch', 'non_banking_institution': 'non-bank-institution', - 'account': 'account', 'currency_code': 'currency-code', 'account_name': 'account-name', - 'client_number': 'client-number', 'personal_account_type': 'personal-account-type', 'opened': 'opened', - 'closed': 'closed', 'balance': 'balance', 'status_code': 'status-code', 'beneficiary': 'beneficiary', - 'beneficiary_comment': 'beneficiary-comment', 'comments': 'comments'} - -t_person_mapping = {'misp_name': 'person', 'comments': 'text', 'first_name': 'first-name', 'middle_name': 'middle-name', - 'last_name': 'last-name', 'title': 'title', 'mothers_name': 'mothers-name', 'alias': 'alias', - 'birthdate': 'date-of-birth', 'birth_place': 'place-of-birth', 'gender': 'gender', 'nationality1': 'nationality', - 'passport_number': 'passport-number', 'passport_country': 'passport-country', 'ssn': 'social-security-number', - 'id_number': 'identity-card-number'} - -location_mapping = {'misp_name': 'geolocation', 'city': 'city', 'state': 'region', 'country_code': 'country', 'address': 'address', - 'zip': 'zipcode'} - -t_entity_mapping = {'misp_name': 'legal-entity', 'name': 'name', 'business': 'business', 'commercial_name': 'commercial-name', - 'phone': 'phone-number', 'incorporation_legal_form': 'legal-form', 'incorporation_number': 'registration-number'} - -goAMLmapping = {'from_account': t_account_mapping, 'to_account': t_account_mapping, 't_person': t_person_mapping, - 'from_person': t_person_mapping, 'to_person': t_person_mapping, 'reporting_person': t_person_mapping, - 'from_entity': t_entity_mapping, 'to_entity': t_entity_mapping, - 'location': location_mapping, 'address': location_mapping, - 'transaction': {'misp_name': 'transaction', 'transactionnumber': 'transaction-number', 'date_transaction': 'date', - 'transaction_location': 'location', 'transmode_code': 'transmode-code', 'amount_local': 'amount', - 'transmode_comment': 'transmode-comment', 'date_posting': 'date-posting', 'teller': 'teller', - 'authorized': 'authorized', 'transaction_description': 'text'}} - -nodes_to_ignore = ['addresses', 'signatory'] -relationship_to_keep = ['signatory', 't_from', 't_from_my_client', 't_to', 't_to_my_client', 'address'] - - -class GoAmlParser(): +mispattributes = { + "inputSource": ["file"], + "output": ["MISP objects"], + "format": "misp_standard", +} + +t_from_objects = { + "nodes": ["from_person", "from_account", "from_entity"], + "leaves": ["from_funds_code", "from_country"], +} +t_to_objects = { + "nodes": ["to_person", "to_account", "to_entity"], + "leaves": ["to_funds_code", "to_country"], +} +t_person_objects = { + "nodes": ["addresses"], + "leaves": [ + "first_name", + "middle_name", + "last_name", + "gender", + "title", + "mothers_name", + "birthdate", + "passport_number", + "passport_country", + "id_number", + "birth_place", + "alias", + "nationality1", + ], +} +t_account_objects = { + "nodes": ["signatory"], + "leaves": [ + "institution_name", + "institution_code", + "swift", + "branch", + "non_banking_insitution", + "account", + "currency_code", + "account_name", + "iban", + "client_number", + "opened", + "closed", + "personal_account_type", + "balance", + "date_balance", + "status_code", + "beneficiary", + "beneficiary_comment", + "comments", + ], +} +entity_objects = { + "nodes": ["addresses"], + "leaves": [ + "name", + "commercial_name", + "incorporation_legal_form", + "incorporation_number", + "business", + "phone", + ], +} + +goAMLobjects = { + "report": { + "nodes": ["reporting_person", "location"], + "leaves": [ + "rentity_id", + "submission_code", + "report_code", + "submission_date", + "currency_code_local", + ], + }, + "reporting_person": { + "nodes": ["addresses"], + "leaves": ["first_name", "middle_name", "last_name", "title"], + }, + "location": { + "nodes": [], + "leaves": ["address_type", "address", "city", "zip", "country_code", "state"], + }, + "transaction": { + "nodes": ["t_from", "t_from_my_client", "t_to", "t_to_my_client"], + "leaves": [ + "transactionnumber", + "transaction_location", + "date_transaction", + "transmode_code", + "amount_local", + ], + }, + "t_from": t_from_objects, + "t_from_my_client": t_from_objects, + "t_to": t_to_objects, + "t_to_my_client": t_to_objects, + "addresses": {"nodes": ["address"], "leaves": []}, + "address": { + "nodes": [], + "leaves": ["address_type", "address", "city", "zip", "country_code", "state"], + }, + "from_person": t_person_objects, + "to_person": t_person_objects, + "t_person": t_person_objects, + "from_account": t_account_objects, + "to_account": t_account_objects, + "signatory": {"nodes": ["t_person"], "leaves": []}, + "from_entity": entity_objects, + "to_entity": entity_objects, +} + +t_account_mapping = { + "misp_name": "bank-account", + "institution_name": "institution-name", + "institution_code": "institution-code", + "iban": "iban", + "swift": "swift", + "branch": "branch", + "non_banking_institution": "non-bank-institution", + "account": "account", + "currency_code": "currency-code", + "account_name": "account-name", + "client_number": "client-number", + "personal_account_type": "personal-account-type", + "opened": "opened", + "closed": "closed", + "balance": "balance", + "status_code": "status-code", + "beneficiary": "beneficiary", + "beneficiary_comment": "beneficiary-comment", + "comments": "comments", +} + +t_person_mapping = { + "misp_name": "person", + "comments": "text", + "first_name": "first-name", + "middle_name": "middle-name", + "last_name": "last-name", + "title": "title", + "mothers_name": "mothers-name", + "alias": "alias", + "birthdate": "date-of-birth", + "birth_place": "place-of-birth", + "gender": "gender", + "nationality1": "nationality", + "passport_number": "passport-number", + "passport_country": "passport-country", + "ssn": "social-security-number", + "id_number": "identity-card-number", +} + +location_mapping = { + "misp_name": "geolocation", + "city": "city", + "state": "region", + "country_code": "country", + "address": "address", + "zip": "zipcode", +} + +t_entity_mapping = { + "misp_name": "legal-entity", + "name": "name", + "business": "business", + "commercial_name": "commercial-name", + "phone": "phone-number", + "incorporation_legal_form": "legal-form", + "incorporation_number": "registration-number", +} + +goAMLmapping = { + "from_account": t_account_mapping, + "to_account": t_account_mapping, + "t_person": t_person_mapping, + "from_person": t_person_mapping, + "to_person": t_person_mapping, + "reporting_person": t_person_mapping, + "from_entity": t_entity_mapping, + "to_entity": t_entity_mapping, + "location": location_mapping, + "address": location_mapping, + "transaction": { + "misp_name": "transaction", + "transactionnumber": "transaction-number", + "date_transaction": "date", + "transaction_location": "location", + "transmode_code": "transmode-code", + "amount_local": "amount", + "transmode_comment": "transmode-comment", + "date_posting": "date-posting", + "teller": "teller", + "authorized": "authorized", + "transaction_description": "text", + }, +} + +nodes_to_ignore = ["addresses", "signatory"] +relationship_to_keep = [ + "signatory", + "t_from", + "t_from_my_client", + "t_to", + "t_to_my_client", + "address", +] + + +class GoAmlParser: def __init__(self): self.misp_event = MISPEvent() @@ -95,13 +252,13 @@ def read_xml(self, data): def parse_xml(self): self.first_itteration() - for t in self.tree.findall('transaction'): - self.itterate(t, 'transaction') + for t in self.tree.findall("transaction"): + self.itterate(t, "transaction") def first_itteration(self): - submission_date = self.tree.find('submission_date').text.split('+')[0] + submission_date = self.tree.find("submission_date").text.split("+")[0] self.misp_event.timestamp = int(time.mktime(time.strptime(submission_date, "%Y-%m-%dT%H:%M:%S"))) - for node in goAMLobjects['report']['nodes']: + for node in goAMLobjects["report"]["nodes"]: element = self.tree.find(node) if element is not None: self.itterate(element, element.tag) @@ -113,15 +270,18 @@ def itterate(self, tree, aml_type, referencing_uuid=None, relationship_type=None if aml_type not in nodes_to_ignore: try: mapping = goAMLmapping[aml_type] - misp_object = MISPObject(name=mapping['misp_name']) - for leaf in objects['leaves']: + misp_object = MISPObject(name=mapping["misp_name"]) + for leaf in objects["leaves"]: element = tree.find(leaf) if element is not None: object_relation = mapping[element.tag] - attribute = {'object_relation': object_relation, 'value': element.text} + attribute = { + "object_relation": object_relation, + "value": element.text, + } misp_object.add_attribute(**attribute) - if aml_type == 'transaction': - for node in objects['nodes']: + if aml_type == "transaction": + for node in objects["nodes"]: element = tree.find(node) if element is not None: self.fill_transaction(element, element.tag, misp_object) @@ -133,29 +293,43 @@ def itterate(self, tree, aml_type, referencing_uuid=None, relationship_type=None referencing_object.add_reference(referenced_uuid, rel, None, **last_object) except KeyError: pass - for node in objects['nodes']: + for node in objects["nodes"]: element = tree.find(node) if element is not None: tag = element.tag if tag in relationship_to_keep: - rel = tag[2:] if tag.startswith('t_') else tag - self.itterate(element, element.tag, referencing_uuid=referenced_uuid, relationship_type=rel) + rel = tag[2:] if tag.startswith("t_") else tag + self.itterate( + element, + element.tag, + referencing_uuid=referenced_uuid, + relationship_type=rel, + ) @staticmethod def fill_transaction(element, tag, misp_object): - if 't_from' in tag: - from_funds = element.find('from_funds_code').text - from_funds_attribute = {'object_relation': 'from-funds-code', 'value': from_funds} + if "t_from" in tag: + from_funds = element.find("from_funds_code").text + from_funds_attribute = { + "object_relation": "from-funds-code", + "value": from_funds, + } misp_object.add_attribute(**from_funds_attribute) - from_country = element.find('from_country').text - from_country_attribute = {'object_relation': 'from-country', 'value': from_country} + from_country = element.find("from_country").text + from_country_attribute = { + "object_relation": "from-country", + "value": from_country, + } misp_object.add_attribute(**from_country_attribute) - if 't_to' in tag: - to_funds = element.find('to_funds_code').text - to_funds_attribute = {'object_relation': 'to-funds-code', 'value': to_funds} + if "t_to" in tag: + to_funds = element.find("to_funds_code").text + to_funds_attribute = {"object_relation": "to-funds-code", "value": to_funds} misp_object.add_attribute(**to_funds_attribute) - to_country = element.find('to_country').text - to_country_attribute = {'object_relation': 'to-country', 'value': to_country} + to_country = element.find("to_country").text + to_country_attribute = { + "object_relation": "to-country", + "value": to_country, + } misp_object.add_attribute(**to_country_attribute) @@ -163,19 +337,19 @@ def handler(q=False): if q is False: return False request = json.loads(q) - if request.get('data'): - data = base64.b64decode(request['data']).decode('utf-8') + if request.get("data"): + data = base64.b64decode(request["data"]).decode("utf-8") else: - misperrors['error'] = "Unsupported attributes type" + misperrors["error"] = "Unsupported attributes type" return misperrors aml_parser = GoAmlParser() try: aml_parser.read_xml(data) except Exception: - misperrors['error'] = "Impossible to read XML data" + misperrors["error"] = "Impossible to read XML data" return misperrors aml_parser.parse_xml() - r = {'results': {'Object': [obj.to_json() for obj in aml_parser.misp_event.objects]}} + r = {"results": {"Object": [obj.to_json() for obj in aml_parser.misp_event.objects]}} return r @@ -184,5 +358,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/import_mod/import_blueprint.py b/misp_modules/modules/import_mod/import_blueprint.py index 30a721067..af71305df 100755 --- a/misp_modules/modules/import_mod/import_blueprint.py +++ b/misp_modules/modules/import_mod/import_blueprint.py @@ -1,48 +1,40 @@ -import json import base64 -from pymisp import MISPEvent, MISPObject, MISPAttribute +import json + +from pymisp import MISPEvent -misperrors = {'error': 'Error'} +misperrors = {"error": "Error"} userConfig = { - 'number1': { - 'type': 'Integer', - 'regex': '/^[0-4]$/i', - 'errorMessage': 'Expected a number in range [0-4]', - 'message': 'Column number used for value' - }, - 'some_string': { - 'type': 'String', - 'message': 'A text field' - }, - 'boolean_field': { - 'type': 'Boolean', - 'message': 'Boolean field test' + "number1": { + "type": "Integer", + "regex": "/^[0-4]$/i", + "errorMessage": "Expected a number in range [0-4]", + "message": "Column number used for value", }, - 'comment': { - 'type': 'Integer', - 'message': 'Column number used for comment' - } + "some_string": {"type": "String", "message": "A text field"}, + "boolean_field": {"type": "Boolean", "message": "Boolean field test"}, + "comment": {"type": "Integer", "message": "Column number used for comment"}, } mispattributes = { - 'inputSource': ['file', 'paste'], - 'output': ['MISP Format'], - 'format': 'misp_standard' + "inputSource": ["file", "paste"], + "output": ["MISP Format"], + "format": "misp_standard", } moduleinfo = { - 'version': '0.1', - 'author': 'Sami Mokaddem', - 'description': 'Generic blueprint to be copy-pasted to quickly boostrap creation of import module.', - 'module-type': ['import'], - 'name': 'Import Blueprint', - 'logo': '', - 'requirements': [], - 'features': '', - 'references': [], - 'input': '', - 'output': '', + "version": "0.1", + "author": "Sami Mokaddem", + "description": "Generic blueprint to be copy-pasted to quickly boostrap creation of import module.", + "module-type": ["import"], + "name": "Import Blueprint", + "logo": "", + "requirements": [], + "features": "", + "references": [], + "input": "", + "output": "", } moduleconfig = [] @@ -71,23 +63,23 @@ def handler(q=False): def getUploadedData(request): - return base64.b64decode(request['data']).decode('utf8') + return base64.b64decode(request["data"]).decode("utf8") def getPassedConfig(request): - return request['config'] + return request["config"] def introspection(): modulesetup = mispattributes try: userConfig - modulesetup['userConfig'] = userConfig + modulesetup["userConfig"] = userConfig except NameError: pass return modulesetup def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/import_mod/joe_import.py b/misp_modules/modules/import_mod/joe_import.py index 68b41ee0e..a8681025f 100644 --- a/misp_modules/modules/import_mod/joe_import.py +++ b/misp_modules/modules/import_mod/joe_import.py @@ -1,9 +1,10 @@ # -*- coding: utf-8 -*- import base64 import json + from joe_parser import JoeParser -misperrors = {'error': 'Error'} +misperrors = {"error": "Error"} userConfig = { "Import Executable": { "type": "Boolean", @@ -15,20 +16,25 @@ }, } -inputSource = ['file'] +inputSource = ["file"] moduleinfo = { - 'version': '0.2', - 'author': 'Christian Studer', - 'description': 'A module to import data from a Joe Sandbox analysis json report.', - 'module-type': ['import'], - 'name': 'Joe Sandbox Import', - 'logo': 'joesandbox.png', - 'requirements': [], - 'features': 'Module using the new format of modules able to return attributes and objects.\n\nThe module returns the same results as the expansion module [joesandbox_query](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/joesandbox_query.py) using the submission link of the analysis to get the json report.', - 'references': ['https://www.joesecurity.org', 'https://www.joesandbox.com/'], - 'input': 'Json report of a Joe Sandbox analysis.', - 'output': 'MISP attributes & objects parsed from the analysis report.', + "version": "0.2", + "author": "Christian Studer", + "description": "A module to import data from a Joe Sandbox analysis json report.", + "module-type": ["import"], + "name": "Joe Sandbox Import", + "logo": "joesandbox.png", + "requirements": [], + "features": ( + "Module using the new format of modules able to return attributes and objects.\n\nThe module returns the same" + " results as the expansion module" + " [joesandbox_query](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/joesandbox_query.py)" + " using the submission link of the analysis to get the json report." + ), + "references": ["https://www.joesecurity.org", "https://www.joesandbox.com/"], + "input": "Json report of a Joe Sandbox analysis.", + "output": "MISP attributes & objects parsed from the analysis report.", } moduleconfig = [] @@ -43,32 +49,32 @@ def handler(q=False): "mitre_attack": bool(int(q["config"]["Mitre Att&ck"])), } - data = base64.b64decode(q.get('data')).decode('utf-8') + data = base64.b64decode(q.get("data")).decode("utf-8") if not data: - return json.dumps({'success': 0}) + return json.dumps({"success": 0}) joe_parser = JoeParser(config) - joe_parser.parse_data(json.loads(data)['analysis']) + joe_parser.parse_data(json.loads(data)["analysis"]) joe_parser.finalize_results() - return {'results': joe_parser.results} + return {"results": joe_parser.results} def introspection(): modulesetup = {} try: userConfig - modulesetup['userConfig'] = userConfig + modulesetup["userConfig"] = userConfig except NameError: pass try: inputSource - modulesetup['inputSource'] = inputSource + modulesetup["inputSource"] = inputSource except NameError: pass - modulesetup['format'] = 'misp_standard' + modulesetup["format"] = "misp_standard" return modulesetup def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/import_mod/lastline_import.py b/misp_modules/modules/import_mod/lastline_import.py index f8d917330..73dc03cec 100644 --- a/misp_modules/modules/import_mod/lastline_import.py +++ b/misp_modules/modules/import_mod/lastline_import.py @@ -8,7 +8,6 @@ import lastline_api - misperrors = { "error": "Error", } @@ -18,24 +17,32 @@ "type": "String", "errorMessage": "Expected analysis link", "message": "The link to a Lastline analysis", - "required": True + "required": True, } } inputSource = [] moduleinfo = { - 'version': '0.1', - 'author': 'Stefano Ortolani', - 'description': 'Deprecation notice: this module will be deprecated by December 2021, please use vmware_nsx module.\n\nModule to import and parse reports from Lastline analysis links.', - 'module-type': ['import'], - 'name': 'Lastline Import', - 'logo': 'lastline.png', - 'requirements': [], - 'features': 'The module requires a Lastline Portal `username` and `password`.\nThe module uses the new format and it is able to return MISP attributes and objects.\nThe module returns the same results as the [lastline_query](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/lastline_query.py) expansion module.', - 'references': ['https://www.lastline.com'], - 'input': 'Link to a Lastline analysis.', - 'output': 'MISP attributes and objects parsed from the analysis report.', + "version": "0.1", + "author": "Stefano Ortolani", + "description": ( + "Deprecation notice: this module will be deprecated by December 2021, please use vmware_nsx module.\n\nModule" + " to import and parse reports from Lastline analysis links." + ), + "module-type": ["import"], + "name": "Lastline Import", + "logo": "lastline.png", + "requirements": [], + "features": ( + "The module requires a Lastline Portal `username` and `password`.\nThe module uses the new format and it is" + " able to return MISP attributes and objects.\nThe module returns the same results as the" + " [lastline_query](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/lastline_query.py)" + " expansion module." + ), + "references": ["https://www.lastline.com"], + "input": "Link to a Lastline analysis.", + "output": "MISP attributes and objects parsed from the analysis report.", } moduleconfig = [ @@ -92,7 +99,11 @@ def handler(q=False): # Make the API calls try: - api_client = lastline_api.PortalClient(api_url, auth_data, verify_ssl=config.get('verify_ssl', True).lower() in ("true")) + api_client = lastline_api.PortalClient( + api_url, + auth_data, + verify_ssl=config.get("verify_ssl", True).lower() in ("true"), + ) response = api_client.get_progress(task_uuid) if response.get("completed") != 1: raise ValueError("Analysis is not finished yet.") @@ -114,9 +125,7 @@ def handler(q=False): return { "results": { - key: event_dictionary[key] - for key in ("Attribute", "Object", "Tag") - if (key in event and event[key]) + key: event_dictionary[key] for key in ("Attribute", "Object", "Tag") if (key in event and event[key]) } } @@ -139,9 +148,8 @@ def handler(q=False): "config": { **a, "analysis_link": ( - "https://user.lastline.com/portal#/analyst/task/" - "1fcbcb8f7fb400100772d6a7b62f501b/overview" - ) + "https://user.lastline.com/portal#/analyst/task/1fcbcb8f7fb400100772d6a7b62f501b/overview" + ), } } ) @@ -152,9 +160,8 @@ def handler(q=False): "config": { **a, "analysis_link": ( - "https://user.lastline.com/portal#/analyst/task/" - "f3c0ae115d51001017ff8da768fa6049/overview" - ) + "https://user.lastline.com/portal#/analyst/task/f3c0ae115d51001017ff8da768fa6049/overview" + ), } } ) diff --git a/misp_modules/modules/import_mod/mispjson.py b/misp_modules/modules/import_mod/mispjson.py index e42a95c32..5a9395b11 100755 --- a/misp_modules/modules/import_mod/mispjson.py +++ b/misp_modules/modules/import_mod/mispjson.py @@ -1,23 +1,26 @@ -import json import base64 +import json -misperrors = {'error': 'Error'} +misperrors = {"error": "Error"} userConfig = {} -inputSource = ['file'] +inputSource = ["file"] moduleinfo = { - 'version': '0.1', - 'author': 'Richard van den Berg', - 'description': 'Module to import MISP JSON format for merging MISP events.', - 'module-type': ['import'], - 'name': 'MISP JSON Import', - 'logo': '', - 'requirements': [], - 'features': 'The module simply imports MISP Attributes from an other MISP Event in order to merge events together. There is thus no special feature to make it work.', - 'references': [], - 'input': 'MISP Event', - 'output': 'MISP Event attributes', + "version": "0.1", + "author": "Richard van den Berg", + "description": "Module to import MISP JSON format for merging MISP events.", + "module-type": ["import"], + "name": "MISP JSON Import", + "logo": "", + "requirements": [], + "features": ( + "The module simply imports MISP Attributes from an other MISP Event in order to merge events together. There is" + " thus no special feature to make it work." + ), + "references": [], + "input": "MISP Event", + "output": "MISP Event attributes", } moduleconfig = [] @@ -26,12 +29,12 @@ def handler(q=False): if q is False: return False - r = {'results': []} + r = {"results": []} request = json.loads(q) try: - mfile = base64.b64decode(request["data"]).decode('utf-8') + mfile = base64.b64decode(request["data"]).decode("utf-8") misp = json.loads(mfile) - event = misp['response'][0]['Event'] + event = misp["response"][0]["Event"] for a in event["Attribute"]: tmp = {} tmp["values"] = a["value"] @@ -41,7 +44,7 @@ def handler(q=False): tmp["comment"] = a["comment"] if a.get("data"): tmp["data"] = a["data"] - r['results'].append(tmp) + r["results"].append(tmp) except Exception: pass return r @@ -51,23 +54,23 @@ def introspection(): modulesetup = {} try: userConfig - modulesetup['userConfig'] = userConfig + modulesetup["userConfig"] = userConfig except NameError: pass try: inputSource - modulesetup['inputSource'] = inputSource + modulesetup["inputSource"] = inputSource except NameError: pass return modulesetup def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo -if __name__ == '__main__': - x = open('test.json', 'r') +if __name__ == "__main__": + x = open("test.json", "r") r = handler(q=x.read()) print(json.dumps(r)) diff --git a/misp_modules/modules/import_mod/ocr.py b/misp_modules/modules/import_mod/ocr.py index 68c7e6b4e..8e9621a6f 100755 --- a/misp_modules/modules/import_mod/ocr.py +++ b/misp_modules/modules/import_mod/ocr.py @@ -1,35 +1,37 @@ -import sys -import json import base64 -from io import BytesIO - +import json import logging +import sys +from io import BytesIO -log = logging.getLogger('ocr') +log = logging.getLogger("ocr") log.setLevel(logging.DEBUG) ch = logging.StreamHandler(sys.stdout) ch.setLevel(logging.DEBUG) -formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") ch.setFormatter(formatter) log.addHandler(ch) -misperrors = {'error': 'Error'} +misperrors = {"error": "Error"} userConfig = {} -inputSource = ['file'] +inputSource = ["file"] moduleinfo = { - 'version': '0.2', - 'author': 'Alexandre Dulaunoy', - 'description': 'Optical Character Recognition (OCR) module for MISP.', - 'module-type': ['import'], - 'name': 'OCR Import', - 'logo': '', - 'requirements': [], - 'features': 'The module tries to recognize some text from an image and import the result as a freetext attribute, there is then no special feature asked to users to make it work.', - 'references': [], - 'input': 'Image', - 'output': 'freetext MISP attribute', + "version": "0.2", + "author": "Alexandre Dulaunoy", + "description": "Optical Character Recognition (OCR) module for MISP.", + "module-type": ["import"], + "name": "OCR Import", + "logo": "", + "requirements": [], + "features": ( + "The module tries to recognize some text from an image and import the result as a freetext attribute, there is" + " then no special feature asked to users to make it work." + ), + "references": [], + "input": "Image", + "output": "freetext MISP attribute", } moduleconfig = [] @@ -40,29 +42,29 @@ def handler(q=False): try: from PIL import Image except ImportError: - misperrors['error'] = "Please pip(3) install pillow" + misperrors["error"] = "Please pip(3) install pillow" return misperrors try: # Official ImageMagick module from wand.image import Image as WImage except ImportError: - misperrors['error'] = "Please pip(3) install wand" + misperrors["error"] = "Please pip(3) install wand" return misperrors try: from pytesseract import image_to_string except ImportError: - misperrors['error'] = "Please pip(3) install pytesseract" + misperrors["error"] = "Please pip(3) install pytesseract" return misperrors if q is False: return False - r = {'results': []} + r = {"results": []} request = json.loads(q) document = base64.b64decode(request["data"]) document = WImage(blob=document) - if document.format == 'PDF': + if document.format == "PDF": with document as pdf: # Get number of pages pages = len(pdf.sequence) @@ -74,7 +76,7 @@ def handler(q=False): log.debug("Stitching page {}".format(p + 1)) image = img.composite(pdf.sequence[p], top=pdf.height * p, left=0) # Create a png blob - image = img.make_blob('png') + image = img.make_blob("png") log.debug("Final image size is {}x{}".format(pdf.width, pdf.height * (p + 1))) else: image = base64.b64decode(request["data"]) @@ -85,15 +87,15 @@ def handler(q=False): try: im = Image.open(image_file) except IOError: - misperrors['error'] = "Corrupt or not an image file." + misperrors["error"] = "Corrupt or not an image file." return misperrors ocrized = image_to_string(im) freetext = {} - freetext['values'] = ocrized - freetext['types'] = ['freetext'] - r['results'].append(freetext) + freetext["values"] = ocrized + freetext["types"] = ["freetext"] + r["results"].append(freetext) return r @@ -101,22 +103,22 @@ def introspection(): modulesetup = {} try: userConfig - modulesetup['userConfig'] = userConfig + modulesetup["userConfig"] = userConfig except NameError: pass try: inputSource - modulesetup['inputSource'] = inputSource + modulesetup["inputSource"] = inputSource except NameError: pass return modulesetup def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo -if __name__ == '__main__': - x = open('test.json', 'r') +if __name__ == "__main__": + x = open("test.json", "r") handler(q=x.read()) diff --git a/misp_modules/modules/import_mod/openiocimport.py b/misp_modules/modules/import_mod/openiocimport.py index 3d4ce0570..209b49062 100755 --- a/misp_modules/modules/import_mod/openiocimport.py +++ b/misp_modules/modules/import_mod/openiocimport.py @@ -1,35 +1,38 @@ -import json import base64 +import json from pymisp.tools import openioc -misperrors = {'error': 'Error'} +misperrors = {"error": "Error"} userConfig = { - 'not save ioc': { - 'type': 'Boolean', - 'message': 'If you check this box, IOC file will not save as an attachment in MISP' + "not save ioc": { + "type": "Boolean", + "message": "If you check this box, IOC file will not save as an attachment in MISP", + }, + "default tag": { + "type": "String", + "message": 'Add tags spaced by a comma (tlp:white,misp:threat-level="no-risk")', + "validation": "0", }, - 'default tag': { - 'type': 'String', - 'message': 'Add tags spaced by a comma (tlp:white,misp:threat-level="no-risk")', - 'validation': '0' - } } -inputSource = ['file'] +inputSource = ["file"] moduleinfo = { - 'version': '0.1', - 'author': 'Raphaël Vinot', - 'description': 'Module to import OpenIOC packages.', - 'module-type': ['import'], - 'name': 'OpenIOC Import', - 'logo': '', - 'requirements': ['PyMISP'], - 'features': 'The module imports MISP Attributes from OpenIOC packages, there is then no special feature for users to make it work.', - 'references': ['https://www.fireeye.com/blog/threat-research/2013/10/openioc-basics.html'], - 'input': 'OpenIOC packages', - 'output': 'MISP Event attributes', + "version": "0.1", + "author": "Raphaël Vinot", + "description": "Module to import OpenIOC packages.", + "module-type": ["import"], + "name": "OpenIOC Import", + "logo": "", + "requirements": ["PyMISP"], + "features": ( + "The module imports MISP Attributes from OpenIOC packages, there is then no special feature for users to make" + " it work." + ), + "references": ["https://www.fireeye.com/blog/threat-research/2013/10/openioc-basics.html"], + "input": "OpenIOC packages", + "output": "MISP Event attributes", } moduleconfig = [] @@ -41,13 +44,13 @@ def handler(q=False): return False # The return value - r = {'results': []} + r = {"results": []} # Load up that JSON q = json.loads(q) # It's b64 encoded, so decode that stuff - package = base64.b64decode(q.get("data")).decode('utf-8') + package = base64.b64decode(q.get("data")).decode("utf-8") # If something really weird happened if not package: @@ -55,15 +58,17 @@ def handler(q=False): pkg = openioc.load_openioc(package) - if q.get('config'): - if q['config'].get('not save ioc') == "0": - addFile = {"values": [q.get('filename')], - "types": ['attachment'], - "categories": ['Support Tool'], - "data": q.get('data')} + if q.get("config"): + if q["config"].get("not save ioc") == "0": + addFile = { + "values": [q.get("filename")], + "types": ["attachment"], + "categories": ["Support Tool"], + "data": q.get("data"), + } # add tag - if q['config'].get('default tag') is not None: - addFile["tags"] = q['config']['default tag'].split(",") + if q["config"].get("default tag") is not None: + addFile["tags"] = q["config"]["default tag"].split(",") # add file as attachment r["results"].append(addFile) @@ -73,10 +78,11 @@ def handler(q=False): "values": [attrib.value], "types": [attrib.type], "categories": [attrib.category], - "comment": getattr(attrib, 'comment', '')} + "comment": getattr(attrib, "comment", ""), + } # add tag - if q.get('config') and q['config'].get('default tag') is not None: - toAppend["tags"] = q['config']['default tag'].split(",") + if q.get("config") and q["config"].get("default tag") is not None: + toAppend["tags"] = q["config"]["default tag"].split(",") r["results"].append(toAppend) return r @@ -86,17 +92,17 @@ def introspection(): modulesetup = {} try: userConfig - modulesetup['userConfig'] = userConfig + modulesetup["userConfig"] = userConfig except NameError: pass try: inputSource - modulesetup['inputSource'] = inputSource + modulesetup["inputSource"] = inputSource except NameError: pass return modulesetup def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/import_mod/taxii21.py b/misp_modules/modules/import_mod/taxii21.py index fcfd7ac46..3093f34b4 100644 --- a/misp_modules/modules/import_mod/taxii21.py +++ b/misp_modules/modules/import_mod/taxii21.py @@ -1,16 +1,17 @@ """ Import content from a TAXII 2.1 server. """ + import collections import itertools import json import re +from pathlib import Path + import requests import taxii2client import taxii2client.exceptions -from pathlib import Path -from misp_stix_converter import ( - ExternalSTIX2toMISPParser, InternalSTIX2toMISPParser, _is_stix2_from_misp) +from misp_stix_converter import ExternalSTIX2toMISPParser, InternalSTIX2toMISPParser, _is_stix2_from_misp from stix2.v20 import Bundle as Bundle_v20 from stix2.v21 import Bundle as Bundle_v21 @@ -20,29 +21,30 @@ class ConfigError(Exception): Represents an error in the config settings for one invocation of this module. """ + pass -misperrors = {'error': 'Error'} +misperrors = {"error": "Error"} moduleinfo = { - 'version': '0.2', - 'author': 'Abc', - 'description': 'Import content from a TAXII 2.1 server', - 'module-type': ['import'], - 'name': 'TAXII 2.1 Import', - 'logo': '', - 'requirements': ['misp-lib-stix2', 'misp-stix'], - 'features': '', - 'references': [], - 'input': '', - 'output': '', + "version": "0.2", + "author": "Abc", + "description": "Import content from a TAXII 2.1 server", + "module-type": ["import"], + "name": "TAXII 2.1 Import", + "logo": "", + "requirements": ["misp-lib-stix2", "misp-stix"], + "features": "", + "references": [], + "input": "", + "output": "", } mispattributes = { - 'inputSource': [], - 'output': ['MISP objects'], - 'format': 'misp_standard' + "inputSource": [], + "output": ["MISP objects"], + "format": "misp_standard", } @@ -50,42 +52,36 @@ class ConfigError(Exception): "url": { "type": "String", "message": "A TAXII 2.1 collection URL", - "required": True + "required": True, }, "added_after": { "type": "String", - "message": "Lower bound on time the object was uploaded to the TAXII server" - }, - "stix_id": { - "type": "String", - "message": "STIX ID(s) of objects" + "message": "Lower bound on time the object was uploaded to the TAXII server", }, + "stix_id": {"type": "String", "message": "STIX ID(s) of objects"}, "spec_version": { # TAXII 2.1 specific "type": "String", - "message": "STIX version(s) of objects" - }, - "type": { - "type": "String", - "message": "STIX type(s) of objects" + "message": "STIX version(s) of objects", }, + "type": {"type": "String", "message": "STIX type(s) of objects"}, "version": { "type": "String", - "message": 'Version timestamp(s), or "first"/"last"/"all"' + "message": 'Version timestamp(s), or "first"/"last"/"all"', }, # Should we give some user control over this? It will not be allowed to # exceed the admin setting. "STIX object limit": { "type": "Integer", - "message": "Maximum number of STIX objects to process" + "message": "Maximum number of STIX objects to process", }, "username": { "type": "String", - "message": "Username for TAXII server authentication, if necessary" + "message": "Username for TAXII server authentication, if necessary", }, "password": { "type": "String", - "message": "Password for TAXII server authentication, if necessary" - } + "message": "Password for TAXII server authentication, if necessary", + }, } # Paging will be handled transparently by this module, so user-defined @@ -95,9 +91,7 @@ class ConfigError(Exception): # This module will not process more than this number of STIX objects in total # from a TAXII server in one module invocation (across all pages), to limit # resource consumption. -moduleconfig = [ - "stix_object_limit" -] +moduleconfig = ["stix_object_limit"] # In case there is neither an admin nor user setting given. @@ -114,17 +108,20 @@ class ConfigError(Exception): # Collects module config information necessary to perform the TAXII query. -Config = collections.namedtuple("Config", [ - "url", - "added_after", - "id", - "spec_version", - "type", - "version", - "stix_object_limit", - "username", - "password" -]) +Config = collections.namedtuple( + "Config", + [ + "url", + "added_after", + "id", + "spec_version", + "type", + "version", + "stix_object_limit", + "username", + "password", + ], +) def _pymisp_to_json_serializable(obj): @@ -214,16 +211,10 @@ def _get_config(config): raise ConfigError("A TAXII 2.1 collection URL is required.") if admin_stix_object_limit < 1: - raise ConfigError( - "Invalid admin object limit: must be positive: " - + str(admin_stix_object_limit) - ) + raise ConfigError("Invalid admin object limit: must be positive: " + str(admin_stix_object_limit)) if stix_object_limit < 1: - raise ConfigError( - "Invalid object limit: must be positive: " - + str(stix_object_limit) - ) + raise ConfigError("Invalid object limit: must be positive: " + str(stix_object_limit)) if id_: id_ = _normalize_multi_values(id_) @@ -239,16 +230,21 @@ def _get_config(config): if not spec_version: spec_version = "2.1" if spec_version not in ("2.0", "2.1"): - raise ConfigError('Only spec versions "2.0" and "2.1" are valid versions.') + raise ConfigError('Only spec versions "2.0" and "2.1" are valid versions.') if (username and not password) or (not username and password): - raise ConfigError( - 'Both or neither of "username" and "password" are required.' - ) + raise ConfigError('Both or neither of "username" and "password" are required.') config_obj = Config( - url, added_after, id_, spec_version, type_, version_, stix_object_limit, - username, password + url, + added_after, + id_, + spec_version, + type_, + version_, + stix_object_limit, + username, + password, ) return config_obj @@ -263,16 +259,12 @@ def _query_taxii(config): :return: A dict containing a misp-modules response """ - collection = taxii2client.Collection( - config.url, user=config.username, password=config.password - ) + collection = taxii2client.Collection(config.url, user=config.username, password=config.password) # No point in asking for more than our overall limit. page_size = min(_PAGE_SIZE, config.stix_object_limit) - kwargs = { - "per_request": page_size - } + kwargs = {"per_request": page_size} if config.spec_version: kwargs["spec_version"] = config.spec_version @@ -285,21 +277,13 @@ def _query_taxii(config): if config.added_after: kwargs["added_after"] = config.added_after - pages = taxii2client.as_pages( - collection.get_objects, - **kwargs - ) + pages = taxii2client.as_pages(collection.get_objects, **kwargs) # Chain all the objects from all pages together... - all_stix_objects = itertools.chain.from_iterable( - taxii_envelope.get("objects", []) - for taxii_envelope in pages - ) + all_stix_objects = itertools.chain.from_iterable(taxii_envelope.get("objects", []) for taxii_envelope in pages) # And only take the first N objects from that. - limited_stix_objects = itertools.islice( - all_stix_objects, 0, config.stix_object_limit - ) + limited_stix_objects = itertools.islice(all_stix_objects, 0, config.stix_object_limit) # Collect into a list. This is... unfortunate, but I don't think the # converter will work incrementally (will it?). It expects all objects to @@ -310,39 +294,19 @@ def _query_taxii(config): # memory usage. stix_objects = list(limited_stix_objects) - bundle = (Bundle_v21 if config.spec_version == '2.1' else Bundle_v20)( - stix_objects, allow_custom=True - ) + bundle = (Bundle_v21 if config.spec_version == "2.1" else Bundle_v20)(stix_objects, allow_custom=True) - converter = ( - InternalSTIX2toMISPParser() if _is_stix2_from_misp(bundle.objects) - else ExternalSTIX2toMISPParser() - ) + converter = InternalSTIX2toMISPParser() if _is_stix2_from_misp(bundle.objects) else ExternalSTIX2toMISPParser() converter.load_stix_bundle(bundle) converter.parse_stix_bundle(single_event=True) - attributes = [ - _pymisp_to_json_serializable(attr) - for attr in converter.misp_event.attributes - ] - - objects = [ - _pymisp_to_json_serializable(obj) - for obj in converter.misp_event.objects - ] - - tags = [ - _pymisp_to_json_serializable(tag) - for tag in converter.misp_event.tags - ] - - result = { - "results": { - "Attribute": attributes, - "Object": objects, - "Tag": tags - } - } + attributes = [_pymisp_to_json_serializable(attr) for attr in converter.misp_event.attributes] + + objects = [_pymisp_to_json_serializable(obj) for obj in converter.misp_event.objects] + + tags = [_pymisp_to_json_serializable(tag) for tag in converter.misp_event.tags] + + result = {"results": {"Attribute": attributes, "Object": objects, "Tag": tags}} return result @@ -384,5 +348,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/import_mod/testimport.py b/misp_modules/modules/import_mod/testimport.py index 818d72173..695212084 100755 --- a/misp_modules/modules/import_mod/testimport.py +++ b/misp_modules/modules/import_mod/testimport.py @@ -1,42 +1,33 @@ -import json import base64 +import json -misperrors = {'error': 'Error'} +misperrors = {"error": "Error"} userConfig = { - 'number1': { - 'type': 'Integer', - 'regex': '/^[0-4]$/i', - 'errorMessage': 'Expected a number in range [0-4]', - 'message': 'Column number used for value' - }, - 'some_string': { - 'type': 'String', - 'message': 'A text field' + "number1": { + "type": "Integer", + "regex": "/^[0-4]$/i", + "errorMessage": "Expected a number in range [0-4]", + "message": "Column number used for value", }, - 'boolean_field': { - 'type': 'Boolean', - 'message': 'Boolean field test' - }, - 'comment': { - 'type': 'Integer', - 'message': 'Column number used for comment' - } + "some_string": {"type": "String", "message": "A text field"}, + "boolean_field": {"type": "Boolean", "message": "Boolean field test"}, + "comment": {"type": "Integer", "message": "Column number used for comment"}, } -inputSource = ['file', 'paste'] +inputSource = ["file", "paste"] moduleinfo = { - 'version': '0.2', - 'author': 'Andras Iklody', - 'description': 'Simple CSV import tool with mapable columns', - 'module-type': ['import'], - 'name': 'CSV Test Import', - 'logo': '', - 'requirements': [], - 'features': '', - 'references': [], - 'input': '', - 'output': '', + "version": "0.2", + "author": "Andras Iklody", + "description": "Simple CSV import tool with mapable columns", + "module-type": ["import"], + "name": "CSV Test Import", + "logo": "", + "requirements": [], + "features": "", + "references": [], + "input": "", + "output": "", } moduleconfig = [] @@ -45,11 +36,19 @@ def handler(q=False): if q is False: return False - r = {'results': []} + r = {"results": []} request = json.loads(q) request["data"] = base64.b64decode(request["data"]) # fields = ["value", "category", "type", "comment"] - r = {"results": [{"values": ["192.168.56.1"], "types":["ip-src"], "categories": ["Network activity"]}]} + r = { + "results": [ + { + "values": ["192.168.56.1"], + "types": ["ip-src"], + "categories": ["Network activity"], + } + ] + } return r @@ -57,17 +56,17 @@ def introspection(): modulesetup = {} try: userConfig - modulesetup['userConfig'] = userConfig + modulesetup["userConfig"] = userConfig except NameError: pass try: inputSource - modulesetup['inputSource'] = inputSource + modulesetup["inputSource"] = inputSource except NameError: pass return modulesetup def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/import_mod/threatanalyzer_import.py b/misp_modules/modules/import_mod/threatanalyzer_import.py index 0d7643207..b8988d7f9 100755 --- a/misp_modules/modules/import_mod/threatanalyzer_import.py +++ b/misp_modules/modules/import_mod/threatanalyzer_import.py @@ -1,36 +1,40 @@ -''' +""" import define mandatory -''' -import json +""" + import base64 -import re -import zipfile -import ipaddress import io +import ipaddress +import json import logging +import re +import zipfile -misperrors = {'error': 'Error'} +misperrors = {"error": "Error"} userConfig = {} -inputSource = ['file'] +inputSource = ["file"] moduleinfo = { - 'version': '0.10', - 'author': 'Christophe Vandeplas', - 'description': 'Module to import ThreatAnalyzer archive.zip / analysis.json files.', - 'module-type': ['import'], - 'name': 'ThreadAnalyzer Sandbox Import', - 'logo': '', - 'requirements': [], - 'features': 'The module imports MISP Attributes from a ThreatAnalyzer format file. This file can be either ZIP, or JSON format.\nThere is by the way no special feature for users to make the module work.', - 'references': ['https://www.threattrack.com/malware-analysis.aspx'], - 'input': 'ThreatAnalyzer format file', - 'output': 'MISP Event attributes', + "version": "0.10", + "author": "Christophe Vandeplas", + "description": "Module to import ThreatAnalyzer archive.zip / analysis.json files.", + "module-type": ["import"], + "name": "ThreadAnalyzer Sandbox Import", + "logo": "", + "requirements": [], + "features": ( + "The module imports MISP Attributes from a ThreatAnalyzer format file. This file can be either ZIP, or JSON" + " format.\nThere is by the way no special feature for users to make the module work." + ), + "references": ["https://www.threattrack.com/malware-analysis.aspx"], + "input": "ThreatAnalyzer format file", + "output": "MISP Event attributes", } moduleconfig = [] -log = logging.getLogger('misp-modules') +log = logging.getLogger("misp-modules") # FIXME - many hardcoded filters should be migrated to import regexes. See also https://github.com/MISP/MISP/issues/2712 # DISCLAIMER - This module is to be considered as experimental and needs much fine-tuning. @@ -41,32 +45,42 @@ def handler(q=False): if q is False: return False results = [] - zip_starts = 'PK' + zip_starts = "PK" request = json.loads(q) - data = base64.b64decode(request['data']) + data = base64.b64decode(request["data"]) - if data[:len(zip_starts)].decode() == zip_starts: - with zipfile.ZipFile(io.BytesIO(data), 'r') as zf: + if data[: len(zip_starts)].decode() == zip_starts: + with zipfile.ZipFile(io.BytesIO(data), "r") as zf: # unzipped_files = [] modified_files_mapping = {} # pre-process some of the files in the zip for zip_file_name in zf.namelist(): # Get all files in the zip file # find the filenames of the modified_files if re.match(r"Analysis/proc_\d+/modified_files/mapping\.log", zip_file_name): - with zf.open(zip_file_name, mode='r', pwd=None) as fp: + with zf.open(zip_file_name, mode="r", pwd=None) as fp: file_data = fp.read() - for line in file_data.decode("utf-8", 'ignore').split('\n'): + for line in file_data.decode("utf-8", "ignore").split("\n"): if not line: continue - if line.count('|') == 3: - l_fname, l_size, l_md5, l_created = line.split('|') - if line.count('|') == 4: - l_fname, l_size, l_md5, l_sha256, l_created = line.split('|') + if line.count("|") == 3: + l_fname, l_size, l_md5, l_created = line.split("|") + if line.count("|") == 4: + l_fname, l_size, l_md5, l_sha256, l_created = line.split("|") l_fname = cleanup_filepath(l_fname) if l_fname: if l_size == 0: - results.append({'values': l_fname, 'type': 'filename', 'to_ids': True, - 'categories': ['Artifacts dropped', 'Payload delivery'], 'comment': ''}) + results.append( + { + "values": l_fname, + "type": "filename", + "to_ids": True, + "categories": [ + "Artifacts dropped", + "Payload delivery", + ], + "comment": "", + } + ) else: # file is a non empty sample, upload the sample later modified_files_mapping[l_md5] = l_fname @@ -74,138 +88,233 @@ def handler(q=False): # now really process the data for zip_file_name in zf.namelist(): # Get all files in the zip file # print('Processing file: {}'.format(zip_file_name)) - if re.match(r"Analysis/proc_\d+/modified_files/.+\.", zip_file_name) and "mapping.log" not in zip_file_name: - sample_md5 = zip_file_name.split('/')[-1].split('.')[0] + if ( + re.match(r"Analysis/proc_\d+/modified_files/.+\.", zip_file_name) + and "mapping.log" not in zip_file_name + ): + sample_md5 = zip_file_name.split("/")[-1].split(".")[0] if sample_md5 in modified_files_mapping: current_sample_filename = modified_files_mapping[sample_md5] # print("{} maps to {}".format(sample_md5, current_sample_filename)) - with zf.open(zip_file_name, mode='r', pwd=None) as fp: + with zf.open(zip_file_name, mode="r", pwd=None) as fp: file_data = fp.read() - results.append({ - 'values': current_sample_filename, - 'data': base64.b64encode(file_data).decode(), - 'type': 'malware-sample', 'categories': ['Artifacts dropped', 'Payload delivery'], 'to_ids': True, 'comment': ''}) - - if 'Analysis/analysis.json' in zip_file_name: - with zf.open(zip_file_name, mode='r', pwd=None) as fp: + results.append( + { + "values": current_sample_filename, + "data": base64.b64encode(file_data).decode(), + "type": "malware-sample", + "categories": [ + "Artifacts dropped", + "Payload delivery", + ], + "to_ids": True, + "comment": "", + } + ) + + if "Analysis/analysis.json" in zip_file_name: + with zf.open(zip_file_name, mode="r", pwd=None) as fp: file_data = fp.read() - analysis_json = json.loads(file_data.decode('utf-8')) + analysis_json = json.loads(file_data.decode("utf-8")) results += process_analysis_json(analysis_json) try: - sample_filename = analysis_json.get('analysis').get('@filename') + sample_filename = analysis_json.get("analysis").get("@filename") if sample_filename: - with zf.open('sample', mode='r', pwd=None) as fp: + with zf.open("sample", mode="r", pwd=None) as fp: file_data = fp.read() - results.append({ - 'values': sample_filename, - 'data': base64.b64encode(file_data).decode(), - 'type': 'malware-sample', 'categories': ['Payload delivery', 'Artifacts dropped'], 'to_ids': True, 'comment': ''}) + results.append( + { + "values": sample_filename, + "data": base64.b64encode(file_data).decode(), + "type": "malware-sample", + "categories": ["Payload delivery", "Artifacts dropped"], + "to_ids": True, + "comment": "", + } + ) except Exception: # no 'sample' in archive, might be an url analysis, just ignore pass else: try: - results = process_analysis_json(json.loads(data.decode('utf-8'))) + results = process_analysis_json(json.loads(data.decode("utf-8"))) except ValueError: - log.warning('MISP modules {0} failed: uploaded file is not a zip or json file.'.format(request['module'])) - return {'error': 'Uploaded file is not a zip or json file.'} + log.warning("MISP modules {0} failed: uploaded file is not a zip or json file.".format(request["module"])) + return {"error": "Uploaded file is not a zip or json file."} pass # keep only unique entries based on the value field - results = list({v['values']: v for v in results}.values()) - r = {'results': results} + results = list({v["values"]: v for v in results}.values()) + r = {"results": results} return r def process_analysis_json(analysis_json): - if 'analysis' in analysis_json and 'processes' in analysis_json['analysis'] and 'process' in analysis_json['analysis']['processes']: + if ( + "analysis" in analysis_json + and "processes" in analysis_json["analysis"] + and "process" in analysis_json["analysis"]["processes"] + ): # if 'analysis' in analysis_json and '@filename' in analysis_json['analysis']: # sample['values'] = analysis_json['analysis']['@filename'] - for process in analysis_json['analysis']['processes']['process']: + for process in analysis_json["analysis"]["processes"]["process"]: # print_json(process) - if 'connection_section' in process and 'connection' in process['connection_section']: + if "connection_section" in process and "connection" in process["connection_section"]: # compensate for absurd behavior of the data format: if one entry = immediately the dict, if multiple entries = list containing dicts # this will always create a list, even with only one item - if isinstance(process['connection_section']['connection'], dict): - process['connection_section']['connection'] = [process['connection_section']['connection']] + if isinstance(process["connection_section"]["connection"], dict): + process["connection_section"]["connection"] = [process["connection_section"]["connection"]] # iterate over each entry - for connection_section_connection in process['connection_section']['connection']: + for connection_section_connection in process["connection_section"]["connection"]: # compensate for absurd behavior of the data format: if one entry = immediately the dict, if multiple entries = list containing dicts # this will always create a list, even with only one item - for subsection in ['http_command', 'http_header']: + for subsection in ["http_command", "http_header"]: if isinstance(connection_section_connection[subsection], dict): connection_section_connection[subsection] = [connection_section_connection[subsection]] - if 'name_to_ip' in connection_section_connection: # TA 6.1 data format - connection_section_connection['@remote_ip'] = connection_section_connection['name_to_ip']['@result_addresses'] - connection_section_connection['@remote_hostname'] = connection_section_connection['name_to_ip']['@request_name'] - - connection_section_connection['@remote_ip'] = cleanup_ip(connection_section_connection['@remote_ip']) - connection_section_connection['@remote_hostname'] = cleanup_hostname(connection_section_connection['@remote_hostname']) - if connection_section_connection['@remote_ip'] and connection_section_connection['@remote_hostname']: - val = '{}|{}'.format(connection_section_connection['@remote_hostname'], - connection_section_connection['@remote_ip']) + if "name_to_ip" in connection_section_connection: # TA 6.1 data format + connection_section_connection["@remote_ip"] = connection_section_connection["name_to_ip"][ + "@result_addresses" + ] + connection_section_connection["@remote_hostname"] = connection_section_connection["name_to_ip"][ + "@request_name" + ] + + connection_section_connection["@remote_ip"] = cleanup_ip( + connection_section_connection["@remote_ip"] + ) + connection_section_connection["@remote_hostname"] = cleanup_hostname( + connection_section_connection["@remote_hostname"] + ) + if ( + connection_section_connection["@remote_ip"] + and connection_section_connection["@remote_hostname"] + ): + val = "{}|{}".format( + connection_section_connection["@remote_hostname"], + connection_section_connection["@remote_ip"], + ) # print("connection_section_connection hostname|ip: {}|{} IDS:yes".format( # connection_section_connection['@remote_hostname'], # connection_section_connection['@remote_ip']) # ) - yield({'values': val, 'type': 'domain|ip', 'categories': ['Network activity'], 'to_ids': True, 'comment': ''}) - elif connection_section_connection['@remote_ip']: + yield ( + { + "values": val, + "type": "domain|ip", + "categories": ["Network activity"], + "to_ids": True, + "comment": "", + } + ) + elif connection_section_connection["@remote_ip"]: # print("connection_section_connection ip-dst: {} IDS:yes".format( # connection_section_connection['@remote_ip']) # ) - yield({'values': connection_section_connection['@remote_ip'], 'type': 'ip-dst', 'to_ids': True, 'comment': ''}) - elif connection_section_connection['@remote_hostname']: + yield ( + { + "values": connection_section_connection["@remote_ip"], + "type": "ip-dst", + "to_ids": True, + "comment": "", + } + ) + elif connection_section_connection["@remote_hostname"]: # print("connection_section_connection hostname: {} IDS:yes".format( # connection_section_connection['@remote_hostname']) # ) - yield({'values': connection_section_connection['@remote_hostname'], 'type': 'hostname', 'to_ids': True, 'comment': ''}) - if 'http_command' in connection_section_connection: - for http_command in connection_section_connection['http_command']: + yield ( + { + "values": connection_section_connection["@remote_hostname"], + "type": "hostname", + "to_ids": True, + "comment": "", + } + ) + if "http_command" in connection_section_connection: + for http_command in connection_section_connection["http_command"]: # print('connection_section_connection HTTP COMMAND: {}\t{}'.format( # connection_section_connection['http_command']['@method'], # comment # connection_section_connection['http_command']['@url']) # url # ) - val = cleanup_url(http_command['@url']) + val = cleanup_url(http_command["@url"]) if val: - yield({'values': val, 'type': 'url', 'categories': ['Network activity'], 'to_ids': True, 'comment': http_command['@method']}) - - if 'http_header' in connection_section_connection: - for http_header in connection_section_connection['http_header']: - if 'User-Agent:' in http_header['@header']: - val = http_header['@header'][len('User-Agent: '):] - yield({'values': val, 'type': 'user-agent', 'categories': ['Network activity'], 'to_ids': False, 'comment': ''}) - elif 'Host:' in http_header['@header']: - val = http_header['@header'][len('Host: '):] - if ':' in val: + yield ( + { + "values": val, + "type": "url", + "categories": ["Network activity"], + "to_ids": True, + "comment": http_command["@method"], + } + ) + + if "http_header" in connection_section_connection: + for http_header in connection_section_connection["http_header"]: + if "User-Agent:" in http_header["@header"]: + val = http_header["@header"][len("User-Agent: ") :] + yield ( + { + "values": val, + "type": "user-agent", + "categories": ["Network activity"], + "to_ids": False, + "comment": "", + } + ) + elif "Host:" in http_header["@header"]: + val = http_header["@header"][len("Host: ") :] + if ":" in val: try: - val_port = int(val.split(':')[1]) + val_port = int(val.split(":")[1]) except ValueError: val_port = False - val_hostname = cleanup_hostname(val.split(':')[0]) - val_ip = cleanup_ip(val.split(':')[0]) + val_hostname = cleanup_hostname(val.split(":")[0]) + val_ip = cleanup_ip(val.split(":")[0]) if val_hostname and val_port: - val_combined = '{}|{}'.format(val_hostname, val_port) + val_combined = "{}|{}".format(val_hostname, val_port) # print({'values': val_combined, 'type': 'hostname|port', 'to_ids': True, 'comment': ''}) - yield({'values': val_combined, 'type': 'hostname|port', 'categories': ['Network activity'], 'to_ids': True, 'comment': ''}) + yield ( + { + "values": val_combined, + "type": "hostname|port", + "categories": ["Network activity"], + "to_ids": True, + "comment": "", + } + ) elif val_ip and val_port: - val_combined = '{}|{}'.format(val_ip, val_port) + val_combined = "{}|{}".format(val_ip, val_port) # print({'values': val_combined, 'type': 'ip-dst|port', 'to_ids': True, 'comment': ''}) - yield({'values': val_combined, 'type': 'ip-dst|port', 'to_ids': True, 'comment': ''}) + yield ( + { + "values": val_combined, + "type": "ip-dst|port", + "to_ids": True, + "comment": "", + } + ) else: continue val_hostname = cleanup_hostname(val) if val_hostname: # print({'values': val_hostname, 'type': 'hostname', 'to_ids': True, 'comment': ''}) - yield({'values': val_hostname, 'type': 'hostname', 'to_ids': True, 'comment': ''}) + yield ( + { + "values": val_hostname, + "type": "hostname", + "to_ids": True, + "comment": "", + } + ) else: # LATER header not processed pass - if 'filesystem_section' in process and 'create_file' in process['filesystem_section']: - for filesystem_section_create_file in process['filesystem_section']['create_file']: + if "filesystem_section" in process and "create_file" in process["filesystem_section"]: + for filesystem_section_create_file in process["filesystem_section"]["create_file"]: # first skip some items - if filesystem_section_create_file['@create_disposition'] in {'FILE_OPEN_IF'}: + if filesystem_section_create_file["@create_disposition"] in {"FILE_OPEN_IF"}: continue # FIXME - this section is probably not needed considering the 'stored_files stored_created_file' section we process later. # print('CREATE FILE: {}\t{}'.format( @@ -213,8 +322,10 @@ def process_analysis_json(analysis_json): # filesystem_section_create_file['@create_disposition']) # comment - use this to filter out cases # ) - if 'networkoperation_section' in process and 'dns_request_by_addr' in process['networkoperation_section']: - for networkoperation_section_dns_request_by_addr in process['networkoperation_section']['dns_request_by_addr']: + if "networkoperation_section" in process and "dns_request_by_addr" in process["networkoperation_section"]: + for networkoperation_section_dns_request_by_addr in process["networkoperation_section"][ + "dns_request_by_addr" + ]: # FIXME - it's unclear what this section is for. # TODO filter this # print('DNS REQUEST: {}\t{}'.format( @@ -222,78 +333,151 @@ def process_analysis_json(analysis_json): # networkoperation_section_dns_request_by_addr['@result_name']) # hostname # ) # => NOT hostname|ip pass - if 'networkoperation_section' in process and 'dns_request_by_name' in process['networkoperation_section']: - for networkoperation_section_dns_request_by_name in process['networkoperation_section']['dns_request_by_name']: - networkoperation_section_dns_request_by_name['@request_name'] = cleanup_hostname(networkoperation_section_dns_request_by_name['@request_name'].rstrip('.')) - networkoperation_section_dns_request_by_name['@result_addresses'] = cleanup_ip(networkoperation_section_dns_request_by_name['@result_addresses']) - if networkoperation_section_dns_request_by_name['@request_name'] and networkoperation_section_dns_request_by_name['@result_addresses']: - val = '{}|{}'.format(networkoperation_section_dns_request_by_name['@request_name'], - networkoperation_section_dns_request_by_name['@result_addresses']) + if "networkoperation_section" in process and "dns_request_by_name" in process["networkoperation_section"]: + for networkoperation_section_dns_request_by_name in process["networkoperation_section"][ + "dns_request_by_name" + ]: + networkoperation_section_dns_request_by_name["@request_name"] = cleanup_hostname( + networkoperation_section_dns_request_by_name["@request_name"].rstrip(".") + ) + networkoperation_section_dns_request_by_name["@result_addresses"] = cleanup_ip( + networkoperation_section_dns_request_by_name["@result_addresses"] + ) + if ( + networkoperation_section_dns_request_by_name["@request_name"] + and networkoperation_section_dns_request_by_name["@result_addresses"] + ): + val = "{}|{}".format( + networkoperation_section_dns_request_by_name["@request_name"], + networkoperation_section_dns_request_by_name["@result_addresses"], + ) # print("networkoperation_section_dns_request_by_name hostname|ip: {}|{} IDS:yes".format( # networkoperation_section_dns_request_by_name['@request_name'], # networkoperation_section_dns_request_by_name['@result_addresses']) # ) - yield({'values': val, 'type': 'domain|ip', 'categories': ['Network activity'], 'to_ids': True, 'comment': ''}) - elif networkoperation_section_dns_request_by_name['@request_name']: + yield ( + { + "values": val, + "type": "domain|ip", + "categories": ["Network activity"], + "to_ids": True, + "comment": "", + } + ) + elif networkoperation_section_dns_request_by_name["@request_name"]: # print("networkoperation_section_dns_request_by_name hostname: {} IDS:yes".format( # networkoperation_section_dns_request_by_name['@request_name']) # ) - yield({'values': networkoperation_section_dns_request_by_name['@request_name'], 'type': 'hostname', 'to_ids': True, 'comment': ''}) - elif networkoperation_section_dns_request_by_name['@result_addresses']: + yield ( + { + "values": networkoperation_section_dns_request_by_name["@request_name"], + "type": "hostname", + "to_ids": True, + "comment": "", + } + ) + elif networkoperation_section_dns_request_by_name["@result_addresses"]: # this happens when the IP is both in the request_name and result_address. # print("networkoperation_section_dns_request_by_name hostname: {} IDS:yes".format( # networkoperation_section_dns_request_by_name['@result_addresses']) # ) - yield({'values': networkoperation_section_dns_request_by_name['@result_addresses'], 'type': 'ip-dst', 'to_ids': True, 'comment': ''}) - - if 'networkpacket_section' in process and 'connect_to_computer' in process['networkpacket_section']: - for networkpacket_section_connect_to_computer in process['networkpacket_section']['connect_to_computer']: - networkpacket_section_connect_to_computer['@remote_hostname'] = cleanup_hostname(networkpacket_section_connect_to_computer['@remote_hostname']) - networkpacket_section_connect_to_computer['@remote_ip'] = cleanup_ip(networkpacket_section_connect_to_computer['@remote_ip']) - if networkpacket_section_connect_to_computer['@remote_hostname'] and networkpacket_section_connect_to_computer['@remote_ip']: + yield ( + { + "values": networkoperation_section_dns_request_by_name["@result_addresses"], + "type": "ip-dst", + "to_ids": True, + "comment": "", + } + ) + + if "networkpacket_section" in process and "connect_to_computer" in process["networkpacket_section"]: + for networkpacket_section_connect_to_computer in process["networkpacket_section"][ + "connect_to_computer" + ]: + networkpacket_section_connect_to_computer["@remote_hostname"] = cleanup_hostname( + networkpacket_section_connect_to_computer["@remote_hostname"] + ) + networkpacket_section_connect_to_computer["@remote_ip"] = cleanup_ip( + networkpacket_section_connect_to_computer["@remote_ip"] + ) + if ( + networkpacket_section_connect_to_computer["@remote_hostname"] + and networkpacket_section_connect_to_computer["@remote_ip"] + ): # print("networkpacket_section_connect_to_computer hostname|ip: {}|{} IDS:yes COMMENT:port {}".format( # networkpacket_section_connect_to_computer['@remote_hostname'], # networkpacket_section_connect_to_computer['@remote_ip'], # networkpacket_section_connect_to_computer['@remote_port']) # ) - val_combined = "{}|{}".format(networkpacket_section_connect_to_computer['@remote_hostname'], networkpacket_section_connect_to_computer['@remote_ip']) - yield({'values': val_combined, 'type': 'domain|ip', 'to_ids': True, 'comment': ''}) - elif networkpacket_section_connect_to_computer['@remote_hostname']: + val_combined = "{}|{}".format( + networkpacket_section_connect_to_computer["@remote_hostname"], + networkpacket_section_connect_to_computer["@remote_ip"], + ) + yield ( + { + "values": val_combined, + "type": "domain|ip", + "to_ids": True, + "comment": "", + } + ) + elif networkpacket_section_connect_to_computer["@remote_hostname"]: # print("networkpacket_section_connect_to_computer hostname: {} IDS:yes COMMENT:port {}".format( # networkpacket_section_connect_to_computer['@remote_hostname'], # networkpacket_section_connect_to_computer['@remote_port']) # ) - val_combined = "{}|{}".format(networkpacket_section_connect_to_computer['@remote_hostname'], networkpacket_section_connect_to_computer['@remote_port']) - yield({'values': val_combined, 'type': 'hostname|port', 'categories': ['Network activity'], 'to_ids': True, 'comment': ''}) - elif networkpacket_section_connect_to_computer['@remote_ip']: + val_combined = "{}|{}".format( + networkpacket_section_connect_to_computer["@remote_hostname"], + networkpacket_section_connect_to_computer["@remote_port"], + ) + yield ( + { + "values": val_combined, + "type": "hostname|port", + "categories": ["Network activity"], + "to_ids": True, + "comment": "", + } + ) + elif networkpacket_section_connect_to_computer["@remote_ip"]: # print("networkpacket_section_connect_to_computer ip-dst: {} IDS:yes COMMENT:port {}".format( # networkpacket_section_connect_to_computer['@remote_ip'], # networkpacket_section_connect_to_computer['@remote_port']) # ) - val_combined = "{}|{}".format(networkpacket_section_connect_to_computer['@remote_ip'], networkpacket_section_connect_to_computer['@remote_port']) - yield({'values': val_combined, 'type': 'ip-dst|port', 'to_ids': True, 'comment': ''}) - - if 'registry_section' in process and 'create_key' in process['registry_section']: + val_combined = "{}|{}".format( + networkpacket_section_connect_to_computer["@remote_ip"], + networkpacket_section_connect_to_computer["@remote_port"], + ) + yield ( + { + "values": val_combined, + "type": "ip-dst|port", + "to_ids": True, + "comment": "", + } + ) + + if "registry_section" in process and "create_key" in process["registry_section"]: # FIXME this is a complicated section, together with the 'set_value'. # it looks like this section is not ONLY about creating registry keys, # more about accessing a handle to keys (with specific permissions) # maybe we don't want to keep this, in favor of 'set_value' - for create_key in process['registry_section']['create_key']: + for create_key in process["registry_section"]["create_key"]: # print('REG CREATE: {}\t{}'.format( # create_key['@desired_access'], # create_key['@key_name'])) pass - if 'registry_section' in process and 'delete_key' in process['registry_section']: + if "registry_section" in process and "delete_key" in process["registry_section"]: # LATER we probably don't want to keep this. Much pollution. # Maybe for later once we have filtered out this. - for delete_key in process['registry_section']['delete_key']: + for delete_key in process["registry_section"]["delete_key"]: # print('REG DELETE: {}'.format( # delete_key['@key_name']) # ) pass - if 'registry_section' in process and 'set_value' in process['registry_section']: + if "registry_section" in process and "set_value" in process["registry_section"]: # FIXME this is a complicated section, together with the 'create_key'. - for set_value in process['registry_section']['set_value']: + for set_value in process["registry_section"]["set_value"]: # '@data_type' == 'REG_BINARY', # '@data_type' == 'REG_DWORD', # '@data_type' == 'REG_EXPAND_SZ', @@ -301,18 +485,29 @@ def process_analysis_json(analysis_json): # '@data_type' == 'REG_NONE', # '@data_type' == 'REG_QWORD', # '@data_type' == 'REG_SZ', - regkey = cleanup_regkey("{}\\{}".format(set_value['@key_name'], set_value['@value_name'])) - regdata = cleanup_regdata(set_value.get('@data')) + regkey = cleanup_regkey("{}\\{}".format(set_value["@key_name"], set_value["@value_name"])) + regdata = cleanup_regdata(set_value.get("@data")) if not regkey: continue - if set_value['@data_size'] == '0' or not regdata: + if set_value["@data_size"] == "0" or not regdata: # print('registry_section set_value REG SET: {}\t{}\t{}'.format( # set_value['@data_type'], # set_value['@key_name'], # set_value['@value_name']) # ) - yield({'values': regkey, 'type': 'regkey', 'to_ids': True, - 'categories': ['External analysis', 'Persistence mechanism', 'Artifacts dropped'], 'comment': set_value['@data_type']}) + yield ( + { + "values": regkey, + "type": "regkey", + "to_ids": True, + "categories": [ + "External analysis", + "Persistence mechanism", + "Artifacts dropped", + ], + "comment": set_value["@data_type"], + } + ) else: try: # unicode fun... @@ -323,55 +518,109 @@ def process_analysis_json(analysis_json): # set_value['@data']) # ) val = "{}|{}".format(regkey, regdata) - yield({'values': val, 'type': 'regkey|value', 'to_ids': True, - 'categories': ['External analysis', 'Persistence mechanism', 'Artifacts dropped'], 'comment': set_value['@data_type']}) + yield ( + { + "values": val, + "type": "regkey|value", + "to_ids": True, + "categories": [ + "External analysis", + "Persistence mechanism", + "Artifacts dropped", + ], + "comment": set_value["@data_type"], + } + ) except Exception as e: print("EXCEPTION registry_section {}".format(e)) # TODO - maybe we want to handle these later, or not... pass pass - if 'stored_files' in process and 'stored_created_file' in process['stored_files']: - for stored_created_file in process['stored_files']['stored_created_file']: - stored_created_file['@filename'] = cleanup_filepath(stored_created_file['@filename']) - if stored_created_file['@filename']: - if stored_created_file['@filesize'] != '0': - val = '{}|{}'.format(stored_created_file['@filename'], stored_created_file['@md5']) + if "stored_files" in process and "stored_created_file" in process["stored_files"]: + for stored_created_file in process["stored_files"]["stored_created_file"]: + stored_created_file["@filename"] = cleanup_filepath(stored_created_file["@filename"]) + if stored_created_file["@filename"]: + if stored_created_file["@filesize"] != "0": + val = "{}|{}".format( + stored_created_file["@filename"], + stored_created_file["@md5"], + ) # print("stored_created_file filename|md5: {}|{} IDS:yes".format( # stored_created_file['@filename'], # filename # stored_created_file['@md5']) # md5 # ) # => filename|md5 - yield({'values': val, 'type': 'filename|md5', 'to_ids': True, - 'categories': ['Artifacts dropped', 'Payload delivery'], 'comment': ''}) + yield ( + { + "values": val, + "type": "filename|md5", + "to_ids": True, + "categories": [ + "Artifacts dropped", + "Payload delivery", + ], + "comment": "", + } + ) else: # print("stored_created_file filename: {} IDS:yes".format( # stored_created_file['@filename']) # filename # ) # => filename - yield({'values': stored_created_file['@filename'], - 'type': 'filename', 'to_ids': True, - 'categories': ['Artifacts dropped', 'Payload delivery'], 'comment': ''}) - - if 'stored_files' in process and 'stored_modified_file' in process['stored_files']: - for stored_modified_file in process['stored_files']['stored_modified_file']: - stored_modified_file['@filename'] = cleanup_filepath(stored_modified_file['@filename']) - if stored_modified_file['@filename']: - if stored_modified_file['@filesize'] != '0': - val = '{}|{}'.format(stored_modified_file['@filename'], stored_modified_file['@md5']) + yield ( + { + "values": stored_created_file["@filename"], + "type": "filename", + "to_ids": True, + "categories": [ + "Artifacts dropped", + "Payload delivery", + ], + "comment": "", + } + ) + + if "stored_files" in process and "stored_modified_file" in process["stored_files"]: + for stored_modified_file in process["stored_files"]["stored_modified_file"]: + stored_modified_file["@filename"] = cleanup_filepath(stored_modified_file["@filename"]) + if stored_modified_file["@filename"]: + if stored_modified_file["@filesize"] != "0": + val = "{}|{}".format( + stored_modified_file["@filename"], + stored_modified_file["@md5"], + ) # print("stored_modified_file MODIFY FILE: {}\t{}".format( # stored_modified_file['@filename'], # filename # stored_modified_file['@md5']) # md5 # ) # => filename|md5 - yield({'values': val, 'type': 'filename|md5', 'to_ids': True, - 'categories': ['Artifacts dropped', 'Payload delivery'], - 'comment': 'modified'}) + yield ( + { + "values": val, + "type": "filename|md5", + "to_ids": True, + "categories": [ + "Artifacts dropped", + "Payload delivery", + ], + "comment": "modified", + } + ) else: # print("stored_modified_file MODIFY FILE: {}\t{}".format( # stored_modified_file['@filename']) # filename # ) # => filename - yield({'values': stored_modified_file['@filename'], 'type': 'filename', 'to_ids': True, - 'categories': ['Artifacts dropped', 'Payload delivery'], - 'comment': 'modified'}) + yield ( + { + "values": stored_modified_file["@filename"], + "type": "filename", + "to_ids": True, + "categories": [ + "Artifacts dropped", + "Payload delivery", + ], + "comment": "modified", + } + ) def add_file(filename, results, hash, index, filedata=None): @@ -393,7 +642,7 @@ def add_file_zip(): def print_json(data): - print(json.dumps(data, sort_keys=True, indent=4, separators=(',', ': '))) + print(json.dumps(data, sort_keys=True, indent=4, separators=(",", ": "))) def list_in_string(lst, data, regex=False): @@ -408,14 +657,7 @@ def list_in_string(lst, data, regex=False): def cleanup_ip(item): # you should exclude private IP ranges via import regexes - noise_substrings = { - '224.0.0.', - '127.0.0.', - '8.8.8.8', - '8.8.4.4', - '0.0.0.0', - 'NONE' - } + noise_substrings = {"224.0.0.", "127.0.0.", "8.8.8.8", "8.8.4.4", "0.0.0.0", "NONE"} if list_in_string(noise_substrings, item): return None try: @@ -426,11 +668,7 @@ def cleanup_ip(item): def cleanup_hostname(item): - noise_substrings = { - 'wpad', - 'teredo.ipv6.microsoft.com', - 'WIN7SP1-x64-UNP' - } + noise_substrings = {"wpad", "teredo.ipv6.microsoft.com", "WIN7SP1-x64-UNP"} # take away common known bad if list_in_string(noise_substrings, item): return None @@ -444,37 +682,35 @@ def cleanup_hostname(item): def cleanup_url(item): - if item in ['/']: + if item in ["/"]: return None return item def cleanup_filepath(item): noise_substrings = { - '\\AppData\\Local\\GDIPFONTCACHEV1.DAT', - '\\AppData\\Local\\Microsoft\\Internet Explorer\\DOMStore\\', - '\\AppData\\Local\\Microsoft\\Internet Explorer\\Recovery\\High\\', - '\\AppData\\Local\\Microsoft\\Windows\\Caches\\', - '\\AppData\\Local\\Microsoft\\Windows\\Explorer\\thumbcache', - '\\AppData\\Local\\Microsoft\\Windows\\History\\History.', - '\\AppData\\Local\\Microsoft\\Windows\\Temporary Internet Files\\Content.', - '\\AppData\\Local\\Microsoft\\Windows\\WebCache\\', - '\\AppData\\Local\\Temp\\.*tmp$', - '\\AppData\\LocalLow\\Microsoft\\CryptnetUrlCache\\', - '\\AppData\\LocalLow\\Microsoft\\Internet Explorer\\Services\\search_', - '\\AppData\\Roaming\\Microsoft\\Office\\Recent\\', - '\\AppData\\Roaming\\Microsoft\\Windows\\Cookies\\', - '\\AppData\\Roaming\\Microsoft\\Windows\\Recent\\', - 'C:\\ProgramData\\Microsoft\\OfficeSoftwareProtectionPlatform\\Cache\\cache.dat', - 'C:\\Windows\\Prefetch\\', - - '\\AppData\\Roaming\\Adobe\\Acrobat\\9.0\\SharedDataEvents-journal', - '\\AppData\\Roaming\\Adobe\\Acrobat\\9.0\\UserCache.bin', - - '\\AppData\\Roaming\\Macromedia\\Flash Player\\macromedia.com\\support\\flashplayer\\sys\\settings.sol', - '\\AppData\\Roaming\\Adobe\\Flash Player\\NativeCache\\', - 'C:\\Windows\\AppCompat\\Programs\\', - 'C:\\~' # caused by temp file created by MS Office when opening malicious doc/xls/... + "\\AppData\\Local\\GDIPFONTCACHEV1.DAT", + "\\AppData\\Local\\Microsoft\\Internet Explorer\\DOMStore\\", + "\\AppData\\Local\\Microsoft\\Internet Explorer\\Recovery\\High\\", + "\\AppData\\Local\\Microsoft\\Windows\\Caches\\", + "\\AppData\\Local\\Microsoft\\Windows\\Explorer\\thumbcache", + "\\AppData\\Local\\Microsoft\\Windows\\History\\History.", + "\\AppData\\Local\\Microsoft\\Windows\\Temporary Internet Files\\Content.", + "\\AppData\\Local\\Microsoft\\Windows\\WebCache\\", + "\\AppData\\Local\\Temp\\.*tmp$", + "\\AppData\\LocalLow\\Microsoft\\CryptnetUrlCache\\", + "\\AppData\\LocalLow\\Microsoft\\Internet Explorer\\Services\\search_", + "\\AppData\\Roaming\\Microsoft\\Office\\Recent\\", + "\\AppData\\Roaming\\Microsoft\\Windows\\Cookies\\", + "\\AppData\\Roaming\\Microsoft\\Windows\\Recent\\", + "C:\\ProgramData\\Microsoft\\OfficeSoftwareProtectionPlatform\\Cache\\cache.dat", + "C:\\Windows\\Prefetch\\", + "\\AppData\\Roaming\\Adobe\\Acrobat\\9.0\\SharedDataEvents-journal", + "\\AppData\\Roaming\\Adobe\\Acrobat\\9.0\\UserCache.bin", + "\\AppData\\Roaming\\Macromedia\\Flash Player\\macromedia.com\\support\\flashplayer\\sys\\settings.sol", + "\\AppData\\Roaming\\Adobe\\Flash Player\\NativeCache\\", + "C:\\Windows\\AppCompat\\Programs\\", + "C:\\~", # caused by temp file created by MS Office when opening malicious doc/xls/... } if list_in_string(noise_substrings, item): return None @@ -483,26 +719,26 @@ def cleanup_filepath(item): def cleanup_regkey(item): noise_substrings = { - r'\\CurrentVersion\\Explorer\\FileExts\\[a-z\.]+\\OpenWith', - r'\\CurrentVersion\\Explorer\\RecentDocs\\', - r'\\CurrentVersion\\Explorer\\UserAssist\\', - r'\\Local Settings\\Software\\Microsoft\\Windows\\Shell\\Bag', - r'\\Software\\Classes\\CLSID\\', - r'\\Software\\Classes\\Local Settings\\MuiCache\\', - r'\\Software\\Microsoft\\Internet Explorer\\Main\\WindowsSearch', - r'\\Software\\Microsoft\\Office\\[0-9\.]+\\', - r'\\Software\\Microsoft\\Office\\Common\\Smart Tag\\', - r'\\Software\\Microsoft\\OfficeSoftwareProtectionPlatform\\', - r'\\Software\\Microsoft\\Shared Tools\\Panose\\', - r'\\Software\\Microsoft\\Tracing\\', - r'\\Software\\Microsoft\\Tracing\\powershell_RASAPI32\\', - r'\\Software\\Microsoft\\Tracing\\powershell_RASMANCS\\', - r'\\Software\\Microsoft\\Windows\\CurrentVersion\\Action Center\\', - r'\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\RunMRU\\', - r'\\Software\\Microsoft\\Windows\\CurrentVersion\\Installer\\UserData\\', - r'\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings\\', - r'\\System\\CurrentControlSet\\Services\\RdyBoost\\', - r'\\Usage\\SpellingAndGrammarFiles' + r"\\CurrentVersion\\Explorer\\FileExts\\[a-z\.]+\\OpenWith", + r"\\CurrentVersion\\Explorer\\RecentDocs\\", + r"\\CurrentVersion\\Explorer\\UserAssist\\", + r"\\Local Settings\\Software\\Microsoft\\Windows\\Shell\\Bag", + r"\\Software\\Classes\\CLSID\\", + r"\\Software\\Classes\\Local Settings\\MuiCache\\", + r"\\Software\\Microsoft\\Internet Explorer\\Main\\WindowsSearch", + r"\\Software\\Microsoft\\Office\\[0-9\.]+\\", + r"\\Software\\Microsoft\\Office\\Common\\Smart Tag\\", + r"\\Software\\Microsoft\\OfficeSoftwareProtectionPlatform\\", + r"\\Software\\Microsoft\\Shared Tools\\Panose\\", + r"\\Software\\Microsoft\\Tracing\\", + r"\\Software\\Microsoft\\Tracing\\powershell_RASAPI32\\", + r"\\Software\\Microsoft\\Tracing\\powershell_RASMANCS\\", + r"\\Software\\Microsoft\\Windows\\CurrentVersion\\Action Center\\", + r"\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\RunMRU\\", + r"\\Software\\Microsoft\\Windows\\CurrentVersion\\Installer\\UserData\\", + r"\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings\\", + r"\\System\\CurrentControlSet\\Services\\RdyBoost\\", + r"\\Usage\\SpellingAndGrammarFiles", } if list_in_string(noise_substrings, item, regex=True): return None @@ -512,22 +748,26 @@ def cleanup_regkey(item): def cleanup_regdata(item): if not item: return None - item = item.replace('(UNICODE_0x00000000)', '') + item = item.replace("(UNICODE_0x00000000)", "") return item def get_zipped_contents(filename, data, password=None): - with zipfile.ZipFile(io.BytesIO(data), 'r') as zf: + with zipfile.ZipFile(io.BytesIO(data), "r") as zf: unzipped_files = [] if password is not None: password = str.encode(password) # Byte encoded password required for zip_file_name in zf.namelist(): # Get all files in the zip file # print(zip_file_name) - with zf.open(zip_file_name, mode='r', pwd=password) as fp: + with zf.open(zip_file_name, mode="r", pwd=password) as fp: file_data = fp.read() - unzipped_files.append({'values': zip_file_name, - 'data': file_data, - 'comment': 'Extracted from {0}'.format(filename)}) + unzipped_files.append( + { + "values": zip_file_name, + "data": file_data, + "comment": "Extracted from {0}".format(filename), + } + ) # print("{} : {}".format(zip_file_name, len(file_data))) return unzipped_files @@ -536,17 +776,17 @@ def introspection(): modulesetup = {} try: userConfig - modulesetup['userConfig'] = userConfig + modulesetup["userConfig"] = userConfig except NameError: pass try: inputSource - modulesetup['inputSource'] = inputSource + modulesetup["inputSource"] = inputSource except NameError: pass return modulesetup def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/import_mod/url_import.py b/misp_modules/modules/import_mod/url_import.py index 5b4ddda63..4e2001987 100755 --- a/misp_modules/modules/import_mod/url_import.py +++ b/misp_modules/modules/import_mod/url_import.py @@ -1,35 +1,33 @@ -import json import base64 -from pymisp import MISPEvent, MISPObject, MISPAttribute +import json + +from pymisp import MISPEvent, MISPObject from pymisp.tools._psl_faup import PSLFaup as Faup -misperrors = {'error': 'Error'} +misperrors = {"error": "Error"} userConfig = { - 'include_scheme': { - 'type': 'Boolean', - 'message': 'Include scheme' - }, + "include_scheme": {"type": "Boolean", "message": "Include scheme"}, } mispattributes = { - 'inputSource': ['file', 'paste'], - 'output': ['MISP Format'], - 'format': 'misp_standard' + "inputSource": ["file", "paste"], + "output": ["MISP Format"], + "format": "misp_standard", } moduleinfo = { - 'version': '0.1', - 'author': 'Sami Mokaddem', - 'description': 'Simple URL import tool with Faup', - 'module-type': ['import'], - 'name': 'URL Import', - 'logo': '', - 'requirements': [], - 'features': '', - 'references': [], - 'input': '', - 'output': '', + "version": "0.1", + "author": "Sami Mokaddem", + "description": "Simple URL import tool with Faup", + "module-type": ["import"], + "name": "URL Import", + "logo": "", + "requirements": [], + "features": "", + "references": [], + "input": "", + "output": "", } moduleconfig = [] @@ -41,22 +39,22 @@ def generateData(event, data, config): for url in data.splitlines(): fp.decode(url) parsed = fp.get() - obj = MISPObject('url') - obj.add_attribute('url', type='url', value=url) - if parsed['tld'] is not None: - obj.add_attribute('tld', type='text', value=parsed['tld']) - if parsed['subdomain'] is not None: - obj.add_attribute('subdomain', type='text', value=parsed['subdomain']) - if config['include_scheme'] is True: - obj.add_attribute('scheme', type='text', value=parsed['scheme']) - obj.add_attribute('resource_path', type='text', value=parsed['resource_path']) - obj.add_attribute('query_string', type='text', value=parsed['query_string']) - obj.add_attribute('port', type='port', value=parsed['port']) - obj.add_attribute('host', type='hostname', value=parsed['host']) - if parsed['fragment'] is not None: - obj.add_attribute('fragment', type='text', value=parsed['fragment']) - obj.add_attribute('domain_without_tld', type='text', value=parsed['domain_without_tld']) - obj.add_attribute('domain', type='domain', value=parsed['domain']) + obj = MISPObject("url") + obj.add_attribute("url", type="url", value=url) + if parsed["tld"] is not None: + obj.add_attribute("tld", type="text", value=parsed["tld"]) + if parsed["subdomain"] is not None: + obj.add_attribute("subdomain", type="text", value=parsed["subdomain"]) + if config["include_scheme"] is True: + obj.add_attribute("scheme", type="text", value=parsed["scheme"]) + obj.add_attribute("resource_path", type="text", value=parsed["resource_path"]) + obj.add_attribute("query_string", type="text", value=parsed["query_string"]) + obj.add_attribute("port", type="port", value=parsed["port"]) + obj.add_attribute("host", type="hostname", value=parsed["host"]) + if parsed["fragment"] is not None: + obj.add_attribute("fragment", type="text", value=parsed["fragment"]) + obj.add_attribute("domain_without_tld", type="text", value=parsed["domain_without_tld"]) + obj.add_attribute("domain", type="domain", value=parsed["domain"]) event.objects.append(obj) @@ -72,26 +70,26 @@ def handler(q=False): def getUploadedData(request): - return base64.b64decode(request['data']).decode('utf8') + return base64.b64decode(request["data"]).decode("utf8") def getPassedConfig(request): for k, v in userConfig.items(): - if v['type'] == 'Boolean': - request['config'][k] = True if request['config'][k] == '1' else False - return request['config'] + if v["type"] == "Boolean": + request["config"][k] = True if request["config"][k] == "1" else False + return request["config"] def introspection(): modulesetup = mispattributes try: userConfig - modulesetup['userConfig'] = userConfig + modulesetup["userConfig"] = userConfig except NameError: pass return modulesetup def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/import_mod/vmray_import.py b/misp_modules/modules/import_mod/vmray_import.py index ed196bb55..4cc06116b 100644 --- a/misp_modules/modules/import_mod/vmray_import.py +++ b/misp_modules/modules/import_mod/vmray_import.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -''' +""" Import VMRay results. This version supports import from different analyze jobs, starting from one sample @@ -11,63 +11,64 @@ You can automate this by setting the PyMISP example script 'vmray_automation' as a cron job -''' +""" import json -from _vmray.parser import VMRayParser, VMRayParseError +from _vmray.parser import VMRayParseError, VMRayParser - -misperrors = {'error': 'Error'} +misperrors = {"error": "Error"} moduleinfo = { - 'version': '0.4', - 'author': 'Jens Thom (VMRay), Koen van Impe', - 'description': 'Module to import VMRay (VTI) results.', - 'module-type': ['import'], - 'name': 'VMRay API Import', - 'logo': 'vmray.png', - 'requirements': ['vmray_rest_api'], - 'features': 'The module imports MISP Attributes from VMRay format, using the VMRay api.\nUsers should then provide as the module configuration the API Key as well as the server url in order to fetch their data to import.', - 'references': ['https://www.vmray.com/'], - 'input': 'VMRay format', - 'output': 'MISP Event attributes', + "version": "0.4", + "author": "Jens Thom (VMRay), Koen van Impe", + "description": "Module to import VMRay (VTI) results.", + "module-type": ["import"], + "name": "VMRay API Import", + "logo": "vmray.png", + "requirements": ["vmray_rest_api"], + "features": ( + "The module imports MISP Attributes from VMRay format, using the VMRay api.\nUsers should then provide as the" + " module configuration the API Key as well as the server url in order to fetch their data to import." + ), + "references": ["https://www.vmray.com/"], + "input": "VMRay format", + "output": "MISP Event attributes", } mispattributes = { - 'inputSource': [], - 'output': ['MISP objects'], - 'format': 'misp_standard', + "inputSource": [], + "output": ["MISP objects"], + "format": "misp_standard", } userConfig = { "Sample ID": { "type": "Integer", "errorMessage": "The VMRay sample ID to download the reports", - "required": True + "required": True, }, "VTI": { "type": "Boolean", "message": "Include VMRay Threat Identifiers", - "checked": "True" - }, - "IOCs": { - "type": "Boolean", - "message": "Include IOCs", - "checked": "True" - }, - "Artifacts": { - "type": "Boolean", - "message": "Include other Artifacts" + "checked": "True", }, + "IOCs": {"type": "Boolean", "message": "Include IOCs", "checked": "True"}, + "Artifacts": {"type": "Boolean", "message": "Include other Artifacts"}, "Analysis Details": { "type": "Boolean", "message": "Include Analysis Details", - "checked": "True" - } + "checked": "True", + }, } -moduleconfig = ["apikey", "url", "disable_tags", "disable_misp_objects", "ignore_analysis_finished"] +moduleconfig = [ + "apikey", + "url", + "disable_tags", + "disable_misp_objects", + "ignore_analysis_finished", +] def handler(q=False): @@ -93,5 +94,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/import_mod/vmray_summary_json_import.py b/misp_modules/modules/import_mod/vmray_summary_json_import.py index f0e96d5b4..9505bf4a0 100644 --- a/misp_modules/modules/import_mod/vmray_summary_json_import.py +++ b/misp_modules/modules/import_mod/vmray_summary_json_import.py @@ -1,24 +1,23 @@ import json -from _vmray.parser import VMRayParser, VMRayParseError +from _vmray.parser import VMRayParseError, VMRayParser - -misperrors = {'error': 'Error'} +misperrors = {"error": "Error"} moduleconfig = ["disable_tags"] moduleinfo = { - 'version': '0.1', - 'author': 'VMRay', - 'description': 'Import a VMRay Summary JSON report.', - 'module-type': ['import'], - 'name': 'VMRay Summary JSON Import', - 'logo': '', - 'requirements': [], - 'features': '', - 'references': [], - 'input': '', - 'output': '', + "version": "0.1", + "author": "VMRay", + "description": "Import a VMRay Summary JSON report.", + "module-type": ["import"], + "name": "VMRay Summary JSON Import", + "logo": "", + "requirements": [], + "features": "", + "references": [], + "input": "", + "output": "", } mispattributes = { @@ -31,18 +30,14 @@ "Analysis ID": { "type": "Boolean", "message": "Include Analysis ID", - "checked": "True" + "checked": "True", }, "VTI": { "type": "Boolean", "message": "Include VMRay Threat Identifiers", - "checked": "True" - }, - "IOCs": { - "type": "Boolean", - "message": "Include IOCs", - "checked": "True" + "checked": "True", }, + "IOCs": {"type": "Boolean", "message": "Include IOCs", "checked": "True"}, "Artifacts": { "type": "Boolean", "message": "Include other Artifacts", @@ -54,7 +49,7 @@ "Attach Report": { "type": "Boolean", "message": "Include the original imported file as attachment", - } + }, } diff --git a/poetry.lock b/poetry.lock index 9f026a317..ba76a8e75 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4,10 +4,10 @@ name = "aiohappyeyeballs" version = "2.4.3" description = "Happy Eyeballs for asyncio" -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "aiohappyeyeballs-2.4.3-py3-none-any.whl", hash = "sha256:8a7a83727b2756f394ab2895ea0765a0a8c475e3c71e98d43d76f22b4b435572"}, {file = "aiohappyeyeballs-2.4.3.tar.gz", hash = "sha256:75cf88a15106a5002a8eb1dab212525c00d1f4c0fa96e551c9fbe6f09a621586"}, @@ -17,10 +17,10 @@ files = [ name = "aiohttp" version = "3.11.7" description = "Async http client/server framework (asyncio)" -optional = false +optional = true python-versions = ">=3.9" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "aiohttp-3.11.7-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8bedb1f6cb919af3b6353921c71281b1491f948ca64408871465d889b4ee1b66"}, {file = "aiohttp-3.11.7-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f5022504adab881e2d801a88b748ea63f2a9d130e0b2c430824682a96f6534be"}, @@ -117,10 +117,10 @@ speedups = ["Brotli ; platform_python_implementation == \"CPython\"", "aiodns (> name = "aiosignal" version = "1.3.1" description = "aiosignal: a list of registered asynchronous callbacks" -optional = false +optional = true python-versions = ">=3.7" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, @@ -133,10 +133,10 @@ frozenlist = ">=1.1.0" name = "annotated-types" version = "0.7.0" description = "Reusable constraint types to use with typing.Annotated" -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, @@ -146,10 +146,10 @@ files = [ name = "ansimarkup" version = "2.1.0" description = "Produce colored terminal text with an xml-like markup" -optional = false +optional = true python-versions = ">=3.6" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "ansimarkup-2.1.0-py3-none-any.whl", hash = "sha256:51ab9f3157125c53e93d8fd2e92df37dfa1757c9f2371ed48554e111c7d4401a"}, {file = "ansimarkup-2.1.0.tar.gz", hash = "sha256:7b3e3d93fecc5b64d23a6e8eb96dbc8b0b576a211829d948afb397d241a8c51b"}, @@ -166,10 +166,10 @@ test = ["pytest", "pytest-cov"] name = "antlr4-python3-runtime" version = "4.9.3" description = "ANTLR 4.9.3 runtime for Python 3.7" -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "antlr4-python3-runtime-4.9.3.tar.gz", hash = "sha256:f224469b4168294902bb1efa80a8bf7855f24c99aef99cbefc1bcd3cce77881b"}, ] @@ -178,10 +178,10 @@ files = [ name = "anyio" version = "4.6.2.post1" description = "High level compatibility layer for multiple asynchronous event loop implementations" -optional = false +optional = true python-versions = ">=3.9" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "anyio-4.6.2.post1-py3-none-any.whl", hash = "sha256:6d170c36fba3bdd840c73d3868c1e777e33676a69c3a72cf0a0d5d6d8009b61d"}, {file = "anyio-4.6.2.post1.tar.gz", hash = "sha256:4c8bc31ccdb51c7f7bd251f51c609e038d63e34219b44aa86e47576389880b4c"}, @@ -202,10 +202,10 @@ trio = ["trio (>=0.26.1)"] name = "apiosintds" version = "2.0.3" description = "On demand query API for OSINT.digitalside.it project. You can query for souspicious domains, urls, IPv4 and file hashes." -optional = false +optional = true python-versions = ">3.5.2" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "apiosintDS-2.0.3-py3-none-any.whl", hash = "sha256:e80163a69f8ca0f9fc01bd37b4c6f5937bdc828be8754a79da1da2958dac7493"}, ] @@ -220,10 +220,10 @@ urllib3 = "*" name = "appdirs" version = "1.4.4" description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"}, {file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"}, @@ -233,10 +233,10 @@ files = [ name = "assemblyline-client" version = "4.9.3" description = "Assemblyline v4 client library" -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "assemblyline_client-4.9.3-py2.py3-none-any.whl", hash = "sha256:f14de17f9a5cd922ea78c02cbb8ab616de39b569bc61c2210321948c17870d83"}, ] @@ -254,6 +254,23 @@ socketio-client = "0.5.7.4" [package.extras] test = ["assemblyline", "cart", "pytest"] +[[package]] +name = "asttokens" +version = "3.0.0" +description = "Annotate AST trees with source code positions" +optional = false +python-versions = ">=3.8" +groups = ["test"] +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +files = [ + {file = "asttokens-3.0.0-py3-none-any.whl", hash = "sha256:e3078351a059199dd5138cb1c706e6430c05eff2ff136af5eb4790f9d28932e2"}, + {file = "asttokens-3.0.0.tar.gz", hash = "sha256:0dcd8baa8d62b0c1d118b399b2ddba3c4aff271d0d7a9e0d4c1681c79035bbc7"}, +] + +[package.extras] +astroid = ["astroid (>=2,<4)"] +test = ["astroid (>=2,<4)", "pytest", "pytest-cov", "pytest-xdist"] + [[package]] name = "async-timeout" version = "5.0.1" @@ -271,10 +288,10 @@ files = [ name = "attrs" version = "24.2.0" description = "Classes Without Boilerplate" -optional = false +optional = true python-versions = ">=3.7" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "attrs-24.2.0-py3-none-any.whl", hash = "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2"}, {file = "attrs-24.2.0.tar.gz", hash = "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346"}, @@ -308,10 +325,10 @@ dev = ["backports.zoneinfo ; python_version < \"3.9\"", "freezegun (>=1.0,<2.0)" name = "backoff" version = "1.11.1" description = "Function decoration for backoff and retry" -optional = false +optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "backoff-1.11.1-py2.py3-none-any.whl", hash = "sha256:61928f8fa48d52e4faa81875eecf308eccfb1016b018bb6bd21e05b5d90a96c5"}, {file = "backoff-1.11.1.tar.gz", hash = "sha256:ccb962a2378418c667b3c979b504fdeb7d9e0d29c0579e3b13b86467177728cb"}, @@ -321,10 +338,10 @@ files = [ name = "backports-tarfile" version = "1.2.0" description = "Backport of CPython tarfile module" -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] -markers = "python_version == \"3.9\" or python_version == \"3.11\" or python_version == \"3.10\" or platform_system == \"Linux\" and python_version < \"3.12\" and platform_machine == \"aarch64\"" +markers = "(python_version == \"3.9\" or python_version == \"3.11\" or python_version == \"3.10\" or platform_system == \"Linux\" and python_version < \"3.12\" and platform_machine == \"aarch64\") and extra == \"all\" and python_version < \"3.12\"" files = [ {file = "backports.tarfile-1.2.0-py3-none-any.whl", hash = "sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34"}, {file = "backports_tarfile-1.2.0.tar.gz", hash = "sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991"}, @@ -338,10 +355,10 @@ testing = ["jaraco.test", "pytest (!=8.0.*)", "pytest (>=6,!=8.1.*)", "pytest-ch name = "backscatter" version = "0.2.4" description = "Client to interact with Backscatter.io services." -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "backscatter-0.2.4-py3-none-any.whl", hash = "sha256:afb0efcf5d2551dac953ec4c38fb710b274b8e811775650e02c1ef42cafb14c8"}, {file = "backscatter-0.2.4.tar.gz", hash = "sha256:7a0d1aa3661635de81e2a09b15d53e35cbe399a111cc58a70925f80e6874abd3"}, @@ -357,7 +374,7 @@ version = "4.12.3" description = "Screen-scraping library" optional = false python-versions = ">=3.6.0" -groups = ["main", "unstable"] +groups = ["main"] markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" files = [ {file = "beautifulsoup4-4.12.3-py3-none-any.whl", hash = "sha256:b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed"}, @@ -378,23 +395,71 @@ lxml = ["lxml"] name = "bidict" version = "0.23.1" description = "The bidirectional mapping library for Python." -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "bidict-0.23.1-py3-none-any.whl", hash = "sha256:5dae8d4d79b552a71cbabc7deb25dfe8ce710b17ff41711e13010ead2abfc3e5"}, {file = "bidict-0.23.1.tar.gz", hash = "sha256:03069d763bc387bbd20e7d49914e75fc4132a41937fa3405417e1a5a2d006d71"}, ] +[[package]] +name = "black" +version = "25.1.0" +description = "The uncompromising code formatter." +optional = false +python-versions = ">=3.9" +groups = ["test"] +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +files = [ + {file = "black-25.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:759e7ec1e050a15f89b770cefbf91ebee8917aac5c20483bc2d80a6c3a04df32"}, + {file = "black-25.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0e519ecf93120f34243e6b0054db49c00a35f84f195d5bce7e9f5cfc578fc2da"}, + {file = "black-25.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:055e59b198df7ac0b7efca5ad7ff2516bca343276c466be72eb04a3bcc1f82d7"}, + {file = "black-25.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:db8ea9917d6f8fc62abd90d944920d95e73c83a5ee3383493e35d271aca872e9"}, + {file = "black-25.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a39337598244de4bae26475f77dda852ea00a93bd4c728e09eacd827ec929df0"}, + {file = "black-25.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:96c1c7cd856bba8e20094e36e0f948718dc688dba4a9d78c3adde52b9e6c2299"}, + {file = "black-25.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bce2e264d59c91e52d8000d507eb20a9aca4a778731a08cfff7e5ac4a4bb7096"}, + {file = "black-25.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:172b1dbff09f86ce6f4eb8edf9dede08b1fce58ba194c87d7a4f1a5aa2f5b3c2"}, + {file = "black-25.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4b60580e829091e6f9238c848ea6750efed72140b91b048770b64e74fe04908b"}, + {file = "black-25.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1e2978f6df243b155ef5fa7e558a43037c3079093ed5d10fd84c43900f2d8ecc"}, + {file = "black-25.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b48735872ec535027d979e8dcb20bf4f70b5ac75a8ea99f127c106a7d7aba9f"}, + {file = "black-25.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:ea0213189960bda9cf99be5b8c8ce66bb054af5e9e861249cd23471bd7b0b3ba"}, + {file = "black-25.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8f0b18a02996a836cc9c9c78e5babec10930862827b1b724ddfe98ccf2f2fe4f"}, + {file = "black-25.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:afebb7098bfbc70037a053b91ae8437c3857482d3a690fefc03e9ff7aa9a5fd3"}, + {file = "black-25.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:030b9759066a4ee5e5aca28c3c77f9c64789cdd4de8ac1df642c40b708be6171"}, + {file = "black-25.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:a22f402b410566e2d1c950708c77ebf5ebd5d0d88a6a2e87c86d9fb48afa0d18"}, + {file = "black-25.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a1ee0a0c330f7b5130ce0caed9936a904793576ef4d2b98c40835d6a65afa6a0"}, + {file = "black-25.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f3df5f1bf91d36002b0a75389ca8663510cf0531cca8aa5c1ef695b46d98655f"}, + {file = "black-25.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d9e6827d563a2c820772b32ce8a42828dc6790f095f441beef18f96aa6f8294e"}, + {file = "black-25.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:bacabb307dca5ebaf9c118d2d2f6903da0d62c9faa82bd21a33eecc319559355"}, + {file = "black-25.1.0-py3-none-any.whl", hash = "sha256:95e8176dae143ba9097f351d174fdaf0ccd29efb414b362ae3fd72bf0f710717"}, + {file = "black-25.1.0.tar.gz", hash = "sha256:33496d5cd1222ad73391352b4ae8da15253c5de89b93a80b3e2c8d9a19ec2666"}, +] + +[package.dependencies] +click = ">=8.0.0" +mypy-extensions = ">=0.4.3" +packaging = ">=22.0" +pathspec = ">=0.9.0" +platformdirs = ">=2" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""} + +[package.extras] +colorama = ["colorama (>=0.4.3)"] +d = ["aiohttp (>=3.10)"] +jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] +uvloop = ["uvloop (>=0.15.2)"] + [[package]] name = "blockchain" version = "1.4.4" description = "Blockchain API library (v1)" -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "blockchain-1.4.4.tar.gz", hash = "sha256:dbaa3eebb6f81b4245005739da802c571b09f98d97eb66520afd95d9ccafebe2"}, ] @@ -407,10 +472,10 @@ future = "*" name = "cachetools" version = "5.5.0" description = "Extensible memoizing collections and decorators" -optional = false +optional = true python-versions = ">=3.7" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "cachetools-5.5.0-py3-none-any.whl", hash = "sha256:02134e8439cdc2ffb62023ce1debca2944c3f289d66bb17ead3ab3dede74b292"}, {file = "cachetools-5.5.0.tar.gz", hash = "sha256:2cc24fb4cbe39633fb7badd9db9ca6295d766d9c2995f245725a46715d050f2a"}, @@ -420,10 +485,10 @@ files = [ name = "cattrs" version = "24.1.2" description = "Composable complex class support for attrs and dataclasses." -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "cattrs-24.1.2-py3-none-any.whl", hash = "sha256:67c7495b760168d931a10233f979b28dc04daf853b30752246f4f8471c6d68d0"}, {file = "cattrs-24.1.2.tar.gz", hash = "sha256:8028cfe1ff5382df59dd36474a86e02d817b06eaf8af84555441bac915d2ef85"}, @@ -448,10 +513,10 @@ ujson = ["ujson (>=5.7.0)"] name = "censys" version = "2.0.9" description = "An easy-to-use and lightweight API wrapper for Censys APIs (censys.io)." -optional = false +optional = true python-versions = ">=3.6.2,<4.0" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "censys-2.0.9-py3-none-any.whl", hash = "sha256:ffda72c7b3172bf781660838d5f65a8babd9b083afd0aff862a7e335c90fb79a"}, {file = "censys-2.0.9.tar.gz", hash = "sha256:5a062f2b97f806879896c6a2a350fd36cae5724ae240abf0c2de40895b043a61"}, @@ -468,7 +533,7 @@ version = "2024.8.30" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" -groups = ["main", "docs", "test", "unstable"] +groups = ["main", "docs", "test"] markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" files = [ {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"}, @@ -575,7 +640,7 @@ version = "3.4.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false python-versions = ">=3.7.0" -groups = ["main", "docs", "test", "unstable"] +groups = ["main", "docs", "test"] markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" files = [ {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:4f9fc98dad6c2eaa32fc3af1417d95b5e3d08aff968df0cd320066def971f9a6"}, @@ -689,10 +754,10 @@ files = [ name = "clamd" version = "1.0.2" description = "Clamd is a python interface to Clamd (Clamav daemon)." -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "clamd-1.0.2-py2.py3-none-any.whl", hash = "sha256:5c32546b7d1eb00fd6be00a889d79e00fbf980ed082826ccfa369bce3dcff5e7"}, {file = "clamd-1.0.2.tar.gz", hash = "sha256:d82a2fd814684a35a1b31feadafb2e69c8ebde9403613f6bdaa5d877c0f29560"}, @@ -704,12 +769,12 @@ version = "8.1.7" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.7" -groups = ["main", "docs"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +groups = ["main", "docs", "test"] files = [ {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, ] +markers = {main = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"", docs = "platform_system == \"Linux\" and platform_machine == \"aarch64\"", test = "platform_system == \"Linux\" and platform_machine == \"aarch64\""} [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} @@ -718,10 +783,10 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} name = "click-default-group" version = "1.2.4" description = "click_default_group" -optional = false +optional = true python-versions = ">=2.7" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "click_default_group-1.2.4-py2.py3-none-any.whl", hash = "sha256:9b60486923720e7fc61731bdb32b617039aba820e22e1c88766b1125592eaa5f"}, {file = "click_default_group-1.2.4.tar.gz", hash = "sha256:eb3f3c99ec0d456ca6cd2a7f08f7d4e91771bef51b01bdd9580cc6450fe1251e"}, @@ -737,10 +802,10 @@ test = ["pytest"] name = "click-plugins" version = "1.1.1" description = "An extension module for click to enable registering CLI commands via setuptools entry-points." -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "click-plugins-1.1.1.tar.gz", hash = "sha256:46ab999744a9d831159c3411bb0c79346d94a444df9a3a3742e9ed63645f264b"}, {file = "click_plugins-1.1.1-py2.py3-none-any.whl", hash = "sha256:5d262006d3222f5057fd81e1623d4443e41dcda5dc815c06b442aa3c02889fc8"}, @@ -756,10 +821,10 @@ dev = ["coveralls", "pytest (>=3.6)", "pytest-cov", "wheel"] name = "click-repl" version = "0.3.0" description = "REPL plugin for Click" -optional = false +optional = true python-versions = ">=3.6" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "click-repl-0.3.0.tar.gz", hash = "sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9"}, {file = "click_repl-0.3.0-py3-none-any.whl", hash = "sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812"}, @@ -800,7 +865,7 @@ files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] -markers = {main = "platform_system == \"Linux\" and platform_machine == \"aarch64\"", docs = "platform_system == \"Linux\" and platform_machine == \"aarch64\"", test = "sys_platform == \"win32\""} +markers = {main = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"", docs = "platform_system == \"Linux\" and platform_machine == \"aarch64\"", test = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and (sys_platform == \"win32\" or platform_system == \"Windows\")"} [[package]] name = "colorclass" @@ -819,10 +884,10 @@ files = [ name = "commonmark" version = "0.9.1" description = "Python parser for the CommonMark Markdown spec" -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "commonmark-0.9.1-py2.py3-none-any.whl", hash = "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"}, {file = "commonmark-0.9.1.tar.gz", hash = "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60"}, @@ -847,10 +912,10 @@ files = [ name = "configparser" version = "7.1.0" description = "Updated configparser from stdlib for earlier Pythons." -optional = false +optional = true python-versions = ">=3.8" -groups = ["main", "unstable"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "configparser-7.1.0-py3-none-any.whl", hash = "sha256:98e374573c4e10e92399651e3ba1c47a438526d633c44ee96143dec26dad4299"}, {file = "configparser-7.1.0.tar.gz", hash = "sha256:eb82646c892dbdf773dae19c633044d163c3129971ae09b49410a303b8e0a5f7"}, @@ -864,10 +929,10 @@ test = ["pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytes name = "contourpy" version = "1.3.0" description = "Python library for calculating contours of 2D quadrilateral grids" -optional = false +optional = true python-versions = ">=3.9" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "contourpy-1.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:880ea32e5c774634f9fcd46504bf9f080a41ad855f4fef54f5380f5133d343c7"}, {file = "contourpy-1.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:76c905ef940a4474a6289c71d53122a4f77766eef23c03cd57016ce19d0f7b42"}, @@ -1027,10 +1092,10 @@ toml = ["tomli ; python_full_version <= \"3.11.0a6\""] name = "crowdstrike-falconpy" version = "1.4.6" description = "The CrowdStrike Falcon SDK for Python" -optional = false +optional = true python-versions = ">=3.7" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "crowdstrike_falconpy-1.4.6-py3-none-any.whl", hash = "sha256:f927508ed221bca5736f0f4d697d6e070753ac64419cbbde212b5ee463f667ff"}, {file = "crowdstrike_falconpy-1.4.6.tar.gz", hash = "sha256:01d134570d0b3682e01de9474e56cbcda7bc7ba02413dcf32c1f4d9077859fa8"}, @@ -1050,7 +1115,7 @@ description = "cryptography is a package which provides cryptographic recipes an optional = false python-versions = ">=3.7" groups = ["main"] -markers = "(platform_python_implementation != \"PyPy\" or sys_platform == \"linux\" or platform_system != \"Windows\" and platform_system != \"Darwin\") and platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" files = [ {file = "cryptography-43.0.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:bf7a1932ac4176486eab36a19ed4c0492da5d97123f1406cf15e41b05e787d2e"}, {file = "cryptography-43.0.3-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63efa177ff54aec6e1c0aefaa1a241232dcd37413835a9b674b6e3f0ae2bfd3e"}, @@ -1098,10 +1163,10 @@ test-randomorder = ["pytest-randomly"] name = "cybox" version = "2.1.0.21" description = "A Python library for parsing and generating CybOX content." -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "cybox-2.1.0.21-py2.py3-none-any.whl", hash = "sha256:19a588bcdce8f3a088f7d5edd3e8862c11b701bb3d64257b18f3092deb9c3b7a"}, {file = "cybox-2.1.0.21.tar.gz", hash = "sha256:8b12110180aceed0f85f8d6c1860a32a679c261f097d909384a81b3b73ff9716"}, @@ -1116,10 +1181,10 @@ python-dateutil = "*" name = "cycler" version = "0.12.1" description = "Composable style cycles" -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "cycler-0.12.1-py3-none-any.whl", hash = "sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30"}, {file = "cycler-0.12.1.tar.gz", hash = "sha256:88bb128f02ba341da8ef447245a9e138fae777f6a23943da4540077d3601eb1c"}, @@ -1129,6 +1194,19 @@ files = [ docs = ["ipython", "matplotlib", "numpydoc", "sphinx"] tests = ["pytest", "pytest-cov", "pytest-xdist"] +[[package]] +name = "decorator" +version = "5.2.1" +description = "Decorators for Humans" +optional = false +python-versions = ">=3.8" +groups = ["test"] +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +files = [ + {file = "decorator-5.2.1-py3-none-any.whl", hash = "sha256:d316bb415a2d9e2d2b3abcc4084c6502fc09240e292cd76a76afc106a1c8e04a"}, + {file = "decorator-5.2.1.tar.gz", hash = "sha256:65f266143752f734b0a7cc83c46f4618af75b8c5911b00ccb61d0ac9b6da0360"}, +] + [[package]] name = "deprecated" version = "1.2.15" @@ -1152,10 +1230,10 @@ dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "jinja2 (>=3.0.3,<3.1.0)", " name = "dict2xml" version = "1.7.6" description = "Small utility to convert a python dictionary into an XML string" -optional = false +optional = true python-versions = ">=3.5" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "dict2xml-1.7.6-py3-none-any.whl", hash = "sha256:841a0c1720e4bfa121e958b805f1062fccf5af2970e7a1f81d7fa056f49e5065"}, {file = "dict2xml-1.7.6.tar.gz", hash = "sha256:3e4811f4ef7fca86dede6acf382268ff9bc5735a4aa0e21b465f6eb0c4e81732"}, @@ -1168,10 +1246,10 @@ tests = ["noseofyeti[black] (==2.4.9)", "pytest (==8.3.2)"] name = "dnsdb2" version = "1.1.4" description = "Client for DNSDB API version 2 with Flexible Search" -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "dnsdb2-1.1.4.tar.gz", hash = "sha256:428e9808f5e3fcdaeacc40edc9d5d14837a20fa7f11b87543348ef285b87af5a"}, ] @@ -1186,9 +1264,10 @@ test = ["requests-mock"] name = "dnspython" version = "2.7.0" description = "DNS toolkit" -optional = false +optional = true python-versions = ">=3.9" groups = ["main"] +markers = "extra == \"all\"" files = [ {file = "dnspython-2.7.0-py3-none-any.whl", hash = "sha256:b4c34b7d10b51bcc3a5071e7b8dee77939f1e878477eeecc965e9835f63c6c86"}, {file = "dnspython-2.7.0.tar.gz", hash = "sha256:ce9c432eda0dc91cf618a5cedf1a4e142651196bbcd2c80e89ed5a907e5cfaf1"}, @@ -1207,10 +1286,10 @@ wmi = ["wmi (>=1.5.1)"] name = "domaintools-api" version = "2.1.0" description = "DomainTools Official Python API" -optional = false +optional = true python-versions = ">=3.6" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "domaintools_api-2.1.0-py2.py3-none-any.whl", hash = "sha256:44adabd440d92e216e4b38ec795633ba9ee595bbb85a703340da4edf2006e8eb"}, {file = "domaintools_api-2.1.0.tar.gz", hash = "sha256:d2202128a0c6d542e37b7ca6dfa46b972e77fae7032df40d6c65de4d937c86d8"}, @@ -1253,10 +1332,10 @@ files = [ name = "enum-compat" version = "0.0.3" description = "enum/enum34 compatibility package" -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "enum-compat-0.0.3.tar.gz", hash = "sha256:3677daabed56a6f724451d585662253d8fb4e5569845aafa8bb0da36b1a8751e"}, {file = "enum_compat-0.0.3-py3-none-any.whl", hash = "sha256:88091b617c7fc3bbbceae50db5958023c48dc40b50520005aa3bf27f8f7ea157"}, @@ -1266,10 +1345,10 @@ files = [ name = "et-xmlfile" version = "2.0.0" description = "An implementation of lxml.xmlfile for the standard library" -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa"}, {file = "et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54"}, @@ -1282,15 +1361,31 @@ description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" groups = ["main", "test"] -markers = "python_version == \"3.9\" or python_version == \"3.10\" or platform_system == \"Linux\" and python_version <= \"3.10\" and platform_machine == \"aarch64\"" files = [ {file = "exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b"}, {file = "exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"}, ] +markers = {main = "(python_version == \"3.9\" or python_version == \"3.10\" or platform_system == \"Linux\" and python_version <= \"3.10\" and platform_machine == \"aarch64\") and extra == \"all\" and python_version < \"3.11\"", test = "python_version == \"3.9\" or python_version == \"3.10\" or platform_system == \"Linux\" and python_version <= \"3.10\" and platform_machine == \"aarch64\""} [package.extras] test = ["pytest (>=6)"] +[[package]] +name = "executing" +version = "2.2.0" +description = "Get the currently executing AST node of a frame, and other information" +optional = false +python-versions = ">=3.8" +groups = ["test"] +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +files = [ + {file = "executing-2.2.0-py2.py3-none-any.whl", hash = "sha256:11387150cad388d62750327a53d3339fad4888b39a6fe233c3afbb54ecffd3aa"}, + {file = "executing-2.2.0.tar.gz", hash = "sha256:5d108c028108fe2551d1a7b2e8b713341e2cb4fc0aa7dcf966fa4327a5226755"}, +] + +[package.extras] +tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipython", "littleutils", "pytest", "rich ; python_version >= \"3.11\""] + [[package]] name = "extract-msg" version = "0.52.0" @@ -1324,38 +1419,22 @@ readthedocs = ["sphinx-rtd-theme"] name = "ezodf" version = "0.3.2" description = "A Python package to create/manipulate OpenDocumentFormat files." -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "ezodf-0.3.2.tar.gz", hash = "sha256:000da534f689c6d55297a08f9e2ed7eada9810d194d31d164388162fb391122d"}, ] -[[package]] -name = "fake-useragent" -version = "2.0.3" -description = "Up-to-date simple useragent faker with real world database" -optional = false -python-versions = ">=3.9" -groups = ["unstable"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" -files = [ - {file = "fake_useragent-2.0.3-py3-none-any.whl", hash = "sha256:8bae50abb72c309a5b3ae2f01a0b82426613fd5c4e2a04dca9332399ec44daa1"}, - {file = "fake_useragent-2.0.3.tar.gz", hash = "sha256:af86a26ef8229efece8fed529b4aeb5b73747d889b60f01cd477b6f301df46e6"}, -] - -[package.dependencies] -importlib-resources = {version = ">=6.0", markers = "python_version < \"3.10\""} - [[package]] name = "filelock" version = "3.16.1" description = "A platform independent file lock." -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "filelock-3.16.1-py3-none-any.whl", hash = "sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0"}, {file = "filelock-3.16.1.tar.gz", hash = "sha256:c249fbfcd5db47e5e2d6d62198e565475ee65e4831e2561c8e313fa7eb961435"}, @@ -1384,14 +1463,33 @@ mccabe = ">=0.7.0,<0.8.0" pycodestyle = ">=2.12.0,<2.13.0" pyflakes = ">=3.2.0,<3.3.0" +[[package]] +name = "flake8-pyproject" +version = "1.2.3" +description = "Flake8 plug-in loading the configuration from pyproject.toml" +optional = false +python-versions = ">= 3.6" +groups = ["test"] +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +files = [ + {file = "flake8_pyproject-1.2.3-py3-none-any.whl", hash = "sha256:6249fe53545205af5e76837644dc80b4c10037e73a0e5db87ff562d75fb5bd4a"}, +] + +[package.dependencies] +Flake8 = ">=5" +TOMLi = {version = "*", markers = "python_version < \"3.11\""} + +[package.extras] +dev = ["pyTest", "pyTest-cov"] + [[package]] name = "fonttools" version = "4.55.0" description = "Tools to manipulate font files" -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "fonttools-4.55.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:51c029d4c0608a21a3d3d169dfc3fb776fde38f00b35ca11fdab63ba10a16f61"}, {file = "fonttools-4.55.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bca35b4e411362feab28e576ea10f11268b1aeed883b9f22ed05675b1e06ac69"}, @@ -1463,10 +1561,10 @@ woff = ["brotli (>=1.0.1) ; platform_python_implementation == \"CPython\"", "bro name = "frozenlist" version = "1.5.0" description = "A list-like structure which implements collections.abc.MutableSequence" -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "frozenlist-1.5.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5b6a66c18b5b9dd261ca98dffcb826a525334b2f29e7caa54e182255c5f6a65a"}, {file = "frozenlist-1.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d1b3eb7b05ea246510b43a7e53ed1653e55c2121019a97e60cad7efb881a97bb"}, @@ -1566,10 +1664,10 @@ files = [ name = "future" version = "1.0.0" description = "Clean single-source support for Python 3 and 2" -optional = false +optional = true python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" -groups = ["main", "unstable"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "future-1.0.0-py3-none-any.whl", hash = "sha256:929292d34f5872e70396626ef385ec22355a1fae8ad29e1a734c3e43f9fbc216"}, {file = "future-1.0.0.tar.gz", hash = "sha256:bd2968309307861edae1458a4f8a4f3598c03be43b97521076aebf5d94c07b05"}, @@ -1579,10 +1677,10 @@ files = [ name = "geoip2" version = "4.8.1" description = "MaxMind GeoIP2 API" -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "geoip2-4.8.1-py3-none-any.whl", hash = "sha256:9317bd75d899d3d942face75a003e73d39006e7fc6c7f9c3db91ae28fbf6a464"}, {file = "geoip2-4.8.1.tar.gz", hash = "sha256:9aea2eab4b3e6252f47456528ae9c35b104c45277639c13fce1be87c92f84257"}, @@ -1615,40 +1713,14 @@ python-dateutil = ">=2.8.1" [package.extras] dev = ["flake8", "markdown", "twine", "wheel"] -[[package]] -name = "Google-Search-API" -version = "1.1.14" -description = "Search in google" -optional = false -python-versions = "*" -groups = ["unstable"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" -files = [] -develop = false - -[package.dependencies] -beautifulsoup4 = "*" -fake-useragent = "*" -future = "*" -requests = "*" -selenium = ">=2.44.0,<3.0.0" -unidecode = "*" -vcrpy = "*" - -[package.source] -type = "git" -url = "https://github.com/abenassi/Google-Search-API" -reference = "HEAD" -resolved_reference = "546a59cc22d3260c60a2faf9afe6477168ead627" - [[package]] name = "greynoise" version = "2.3.0" description = "Abstraction to interact with GreyNoise API." -optional = false +optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=3.0" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "greynoise-2.3.0-py3-none-any.whl", hash = "sha256:92a9471fa98a9a3c0c9e93a15cb990dab963e2e3f1ceb1a200785906be24d4fd"}, {file = "greynoise-2.3.0.tar.gz", hash = "sha256:b33bf61db840ff3e62a2fd987dfb01fe32d23f23e6fc21b002b214529daf11d8"}, @@ -1671,10 +1743,10 @@ six = "*" name = "h11" version = "0.14.0" description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" -optional = false +optional = true python-versions = ">=3.7" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, @@ -1684,10 +1756,10 @@ files = [ name = "httpcore" version = "1.0.7" description = "A minimal low-level HTTP client." -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "httpcore-1.0.7-py3-none-any.whl", hash = "sha256:a3fff8f43dc260d5bd363d9f9cf1830fa3a458b332856f34282de498ed420edd"}, {file = "httpcore-1.0.7.tar.gz", hash = "sha256:8551cb62a169ec7162ac7be8d4817d561f60e08eaa485234898414bb5a8a0b4c"}, @@ -1707,10 +1779,10 @@ trio = ["trio (>=0.22.0,<1.0)"] name = "httplib2" version = "0.22.0" description = "A comprehensive HTTP client library." -optional = false +optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "httplib2-0.22.0-py3-none-any.whl", hash = "sha256:14ae0a53c1ba8f3d37e9e27cf37eabb0fb9980f435ba405d546948b009dd64dc"}, {file = "httplib2-0.22.0.tar.gz", hash = "sha256:d7a10bc5ef5ab08322488bde8c726eeee5c8618723fdb399597ec58f3d82df81"}, @@ -1723,10 +1795,10 @@ pyparsing = {version = ">=2.4.2,<3.0.0 || >3.0.0,<3.0.1 || >3.0.1,<3.0.2 || >3.0 name = "httpx" version = "0.27.2" description = "The next generation HTTP client." -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "httpx-0.27.2-py3-none-any.whl", hash = "sha256:7bb2708e112d8fdd7829cd4243970f0c223274051cb35ee80c03301ee29a3df0"}, {file = "httpx-0.27.2.tar.gz", hash = "sha256:f7c2be1d2f3c3c3160d441802406b206c2b76f5947b11115e6df10c6c65e66c2"}, @@ -1752,7 +1824,7 @@ version = "3.10" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.6" -groups = ["main", "docs", "test", "unstable"] +groups = ["main", "docs", "test"] markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" files = [ {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"}, @@ -1773,7 +1845,7 @@ files = [ {file = "importlib_metadata-8.5.0-py3-none-any.whl", hash = "sha256:45e54197d28b7a7f1559e60b95e7c567032b602131fbd588f1497f47880aa68b"}, {file = "importlib_metadata-8.5.0.tar.gz", hash = "sha256:71522656f0abace1d072b9e5481a48f07c138e00f079c38c8f883823f9c26bd7"}, ] -markers = {main = "python_version == \"3.9\" or python_version == \"3.11\" or python_version == \"3.10\" or platform_system == \"Linux\" and python_version < \"3.12\" and platform_machine == \"aarch64\"", docs = "python_version == \"3.9\" or platform_system == \"Linux\" and python_version < \"3.10\" and platform_machine == \"aarch64\""} +markers = {main = "extra == \"all\" and python_version == \"3.9\" or extra == \"all\" and python_version == \"3.11\" or extra == \"all\" and python_version == \"3.10\" or extra == \"all\" and platform_system == \"Linux\" and python_version < \"3.12\" and platform_machine == \"aarch64\"", docs = "python_version == \"3.9\" or platform_system == \"Linux\" and python_version < \"3.10\" and platform_machine == \"aarch64\""} [package.dependencies] zipp = ">=3.20" @@ -1791,10 +1863,10 @@ type = ["pytest-mypy"] name = "importlib-resources" version = "6.4.5" description = "Read resources from Python packages" -optional = false +optional = true python-versions = ">=3.8" -groups = ["main", "unstable"] -markers = "python_version == \"3.9\" or platform_system == \"Linux\" and python_version < \"3.10\" and platform_machine == \"aarch64\"" +groups = ["main"] +markers = "python_version == \"3.9\" and extra == \"all\" or platform_system == \"Linux\" and python_version < \"3.10\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "importlib_resources-6.4.5-py3-none-any.whl", hash = "sha256:ac29d5f956f01d5e4bb63102a5a19957f1b9175e45649977264a1416783bb717"}, {file = "importlib_resources-6.4.5.tar.gz", hash = "sha256:980862a1d16c9e147a59603677fa2aa5fd82b87f223b6cb870695bcfce830065"}, @@ -1824,27 +1896,191 @@ files = [ {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] +[[package]] +name = "ipdb" +version = "0.13.13" +description = "IPython-enabled pdb" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +groups = ["test"] +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +files = [ + {file = "ipdb-0.13.13-py3-none-any.whl", hash = "sha256:45529994741c4ab6d2388bfa5d7b725c2cf7fe9deffabdb8a6113aa5ed449ed4"}, + {file = "ipdb-0.13.13.tar.gz", hash = "sha256:e3ac6018ef05126d442af680aad863006ec19d02290561ac88b8b1c0b0cfc726"}, +] + +[package.dependencies] +decorator = {version = "*", markers = "python_version > \"3.6\""} +ipython = {version = ">=7.31.1", markers = "python_version > \"3.6\""} +tomli = {version = "*", markers = "python_version > \"3.6\" and python_version < \"3.11\""} + +[[package]] +name = "ipython" +version = "8.18.1" +description = "IPython: Productive Interactive Computing" +optional = false +python-versions = ">=3.9" +groups = ["test"] +markers = "python_version == \"3.9\" or platform_system == \"Linux\" and python_version < \"3.10\" and platform_machine == \"aarch64\"" +files = [ + {file = "ipython-8.18.1-py3-none-any.whl", hash = "sha256:e8267419d72d81955ec1177f8a29aaa90ac80ad647499201119e2f05e99aa397"}, + {file = "ipython-8.18.1.tar.gz", hash = "sha256:ca6f079bb33457c66e233e4580ebfc4128855b4cf6370dddd73842a9563e8a27"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +decorator = "*" +exceptiongroup = {version = "*", markers = "python_version < \"3.11\""} +jedi = ">=0.16" +matplotlib-inline = "*" +pexpect = {version = ">4.3", markers = "sys_platform != \"win32\""} +prompt-toolkit = ">=3.0.41,<3.1.0" +pygments = ">=2.4.0" +stack-data = "*" +traitlets = ">=5" +typing-extensions = {version = "*", markers = "python_version < \"3.10\""} + +[package.extras] +all = ["black", "curio", "docrepr", "exceptiongroup", "ipykernel", "ipyparallel", "ipywidgets", "matplotlib", "matplotlib (!=3.2.0)", "nbconvert", "nbformat", "notebook", "numpy (>=1.22)", "pandas", "pickleshare", "pytest (<7)", "pytest (<7.1)", "pytest-asyncio (<0.22)", "qtconsole", "setuptools (>=18.5)", "sphinx (>=1.3)", "sphinx-rtd-theme", "stack-data", "testpath", "trio", "typing-extensions"] +black = ["black"] +doc = ["docrepr", "exceptiongroup", "ipykernel", "matplotlib", "pickleshare", "pytest (<7)", "pytest (<7.1)", "pytest-asyncio (<0.22)", "setuptools (>=18.5)", "sphinx (>=1.3)", "sphinx-rtd-theme", "stack-data", "testpath", "typing-extensions"] +kernel = ["ipykernel"] +nbconvert = ["nbconvert"] +nbformat = ["nbformat"] +notebook = ["ipywidgets", "notebook"] +parallel = ["ipyparallel"] +qtconsole = ["qtconsole"] +test = ["pickleshare", "pytest (<7.1)", "pytest-asyncio (<0.22)", "testpath"] +test-extra = ["curio", "matplotlib (!=3.2.0)", "nbformat", "numpy (>=1.22)", "pandas", "pickleshare", "pytest (<7.1)", "pytest-asyncio (<0.22)", "testpath", "trio"] + +[[package]] +name = "ipython" +version = "8.33.0" +description = "IPython: Productive Interactive Computing" +optional = false +python-versions = ">=3.10" +groups = ["test"] +markers = "python_version == \"3.10\"" +files = [ + {file = "ipython-8.33.0-py3-none-any.whl", hash = "sha256:aa5b301dfe1eaf0167ff3238a6825f810a029c9dad9d3f1597f30bd5ff65cc44"}, + {file = "ipython-8.33.0.tar.gz", hash = "sha256:4c3e36a6dfa9e8e3702bd46f3df668624c975a22ff340e96ea7277afbd76217d"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +decorator = "*" +exceptiongroup = {version = "*", markers = "python_version < \"3.11\""} +jedi = ">=0.16" +matplotlib-inline = "*" +pexpect = {version = ">4.3", markers = "sys_platform != \"win32\" and sys_platform != \"emscripten\""} +prompt_toolkit = ">=3.0.41,<3.1.0" +pygments = ">=2.4.0" +stack_data = "*" +traitlets = ">=5.13.0" +typing_extensions = {version = ">=4.6", markers = "python_version < \"3.12\""} + +[package.extras] +all = ["ipython[black,doc,kernel,matplotlib,nbconvert,nbformat,notebook,parallel,qtconsole]", "ipython[test,test-extra]"] +black = ["black"] +doc = ["docrepr", "exceptiongroup", "intersphinx_registry", "ipykernel", "ipython[test]", "matplotlib", "setuptools (>=18.5)", "sphinx (>=1.3)", "sphinx-rtd-theme", "sphinxcontrib-jquery", "tomli ; python_version < \"3.11\"", "typing_extensions"] +kernel = ["ipykernel"] +matplotlib = ["matplotlib"] +nbconvert = ["nbconvert"] +nbformat = ["nbformat"] +notebook = ["ipywidgets", "notebook"] +parallel = ["ipyparallel"] +qtconsole = ["qtconsole"] +test = ["packaging", "pickleshare", "pytest", "pytest-asyncio (<0.22)", "testpath"] +test-extra = ["curio", "ipython[test]", "matplotlib (!=3.2.0)", "nbformat", "numpy (>=1.23)", "pandas", "trio"] + +[[package]] +name = "ipython" +version = "9.0.0" +description = "IPython: Productive Interactive Computing" +optional = false +python-versions = ">=3.11" +groups = ["test"] +markers = "python_version >= \"3.11\"" +files = [ + {file = "ipython-9.0.0-py3-none-any.whl", hash = "sha256:2cce23069b830a54a5b9d3d66ccd6433047c1503a7b9a3b34593c0b5c2c08477"}, + {file = "ipython-9.0.0.tar.gz", hash = "sha256:9368d65b3d4a471e9a698fed3ea486bbf6737e45111e915279c971b77f974397"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +decorator = "*" +ipython-pygments-lexers = "*" +jedi = ">=0.16" +matplotlib-inline = "*" +pexpect = {version = ">4.3", markers = "sys_platform != \"win32\" and sys_platform != \"emscripten\""} +prompt_toolkit = ">=3.0.41,<3.1.0" +pygments = ">=2.4.0" +stack_data = "*" +traitlets = ">=5.13.0" +typing_extensions = {version = ">=4.6", markers = "python_version < \"3.12\""} + +[package.extras] +all = ["ipython[doc,matplotlib,test,test-extra]"] +black = ["black"] +doc = ["docrepr", "exceptiongroup", "intersphinx_registry", "ipykernel", "ipython[test]", "matplotlib", "setuptools (>=18.5)", "sphinx (>=1.3)", "sphinx-rtd-theme", "sphinx_toml (==0.0.4)", "typing_extensions"] +matplotlib = ["matplotlib"] +test = ["packaging", "pytest", "pytest-asyncio (<0.22)", "testpath"] +test-extra = ["curio", "ipython[test]", "matplotlib (!=3.2.0)", "nbformat", "numpy (>=1.23)", "pandas", "trio"] + +[[package]] +name = "ipython-pygments-lexers" +version = "1.1.1" +description = "Defines a variety of Pygments lexers for highlighting IPython code." +optional = false +python-versions = ">=3.8" +groups = ["test"] +markers = "python_version >= \"3.11\"" +files = [ + {file = "ipython_pygments_lexers-1.1.1-py3-none-any.whl", hash = "sha256:a9462224a505ade19a605f71f8fa63c2048833ce50abc86768a0d81d876dc81c"}, + {file = "ipython_pygments_lexers-1.1.1.tar.gz", hash = "sha256:09c0138009e56b6854f9535736f4171d855c8c08a563a0dcd8022f78355c7e81"}, +] + +[package.dependencies] +pygments = "*" + [[package]] name = "isodate" version = "0.7.2" description = "An ISO 8601 date/time/duration parser and formatter" -optional = false +optional = true python-versions = ">=3.7" groups = ["main"] -markers = "python_version == \"3.9\" or python_version == \"3.10\" or platform_system == \"Linux\" and python_version <= \"3.10\" and platform_machine == \"aarch64\"" +markers = "(python_version == \"3.9\" or python_version == \"3.10\" or platform_system == \"Linux\" and python_version <= \"3.10\" and platform_machine == \"aarch64\") and extra == \"all\" and python_version < \"3.11\"" files = [ {file = "isodate-0.7.2-py3-none-any.whl", hash = "sha256:28009937d8031054830160fce6d409ed342816b543597cece116d966c6d99e15"}, {file = "isodate-0.7.2.tar.gz", hash = "sha256:4cd1aa0f43ca76f4a6c6c0292a85f40b35ec2e43e315b59f06e6d32171a953e6"}, ] +[[package]] +name = "isort" +version = "6.0.1" +description = "A Python utility / library to sort Python imports." +optional = false +python-versions = ">=3.9.0" +groups = ["test"] +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +files = [ + {file = "isort-6.0.1-py3-none-any.whl", hash = "sha256:2dc5d7f65c9678d94c88dfc29161a320eec67328bc97aad576874cb4be1e9615"}, + {file = "isort-6.0.1.tar.gz", hash = "sha256:1cb5df28dfbc742e490c5e41bad6da41b805b0a8be7bc93cd0fb2a8a890ac450"}, +] + +[package.extras] +colors = ["colorama"] +plugins = ["setuptools"] + [[package]] name = "jaraco-classes" version = "3.4.0" description = "Utility functions for Python class constructs" -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "jaraco.classes-3.4.0-py3-none-any.whl", hash = "sha256:f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790"}, {file = "jaraco.classes-3.4.0.tar.gz", hash = "sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd"}, @@ -1861,10 +2097,10 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-ena name = "jaraco-context" version = "6.0.1" description = "Useful decorators and context managers" -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "jaraco.context-6.0.1-py3-none-any.whl", hash = "sha256:f797fc481b490edb305122c9181830a3a5b76d84ef6d1aef2fb9b47ab956f9e4"}, {file = "jaraco_context-6.0.1.tar.gz", hash = "sha256:9bae4ea555cf0b14938dc0aee7c9f32ed303aa20a3b73e7dc80111628792d1b3"}, @@ -1881,10 +2117,10 @@ test = ["portend", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-c name = "jaraco-functools" version = "4.1.0" description = "Functools like those found in stdlib" -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "jaraco.functools-4.1.0-py3-none-any.whl", hash = "sha256:ad159f13428bc4acbf5541ad6dec511f91573b90fba04df61dafa2a1231cf649"}, {file = "jaraco_functools-4.1.0.tar.gz", hash = "sha256:70f7e0e2ae076498e212562325e805204fc092d7b4c17e0e86c959e249701a9d"}, @@ -1905,10 +2141,10 @@ type = ["pytest-mypy"] name = "jbxapi" version = "3.23.0" description = "API for Joe Sandbox" -optional = false +optional = true python-versions = "!=3.0,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "jbxapi-3.23.0-py2.py3-none-any.whl", hash = "sha256:bf50e59ce542013bcd5ec9dd6a1c23f331301b053cce27296c6149af7bbc65e2"}, {file = "jbxapi-3.23.0.tar.gz", hash = "sha256:200590caaa5cfb64ffb36388e6af64d9d9dd83be02155e4a0c64ccbc1cba0b04"}, @@ -1918,14 +2154,35 @@ files = [ pyzipper = {version = ">=0.3.1", markers = "python_version >= \"3.5\""} requests = ">=2.18.4,<3" +[[package]] +name = "jedi" +version = "0.19.2" +description = "An autocompletion tool for Python that can be used for text editors." +optional = false +python-versions = ">=3.6" +groups = ["test"] +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +files = [ + {file = "jedi-0.19.2-py2.py3-none-any.whl", hash = "sha256:a8ef22bde8490f57fe5c7681a3c83cb58874daf72b4784de3cce5b6ef6edb5b9"}, + {file = "jedi-0.19.2.tar.gz", hash = "sha256:4770dc3de41bde3966b02eb84fbcf557fb33cce26ad23da12c742fb50ecb11f0"}, +] + +[package.dependencies] +parso = ">=0.8.4,<0.9.0" + +[package.extras] +docs = ["Jinja2 (==2.11.3)", "MarkupSafe (==1.1.1)", "Pygments (==2.8.1)", "alabaster (==0.7.12)", "babel (==2.9.1)", "chardet (==4.0.0)", "commonmark (==0.8.1)", "docutils (==0.17.1)", "future (==0.18.2)", "idna (==2.10)", "imagesize (==1.2.0)", "mock (==1.0.1)", "packaging (==20.9)", "pyparsing (==2.4.7)", "pytz (==2021.1)", "readthedocs-sphinx-ext (==2.1.4)", "recommonmark (==0.5.0)", "requests (==2.25.1)", "six (==1.15.0)", "snowballstemmer (==2.1.0)", "sphinx (==1.8.5)", "sphinx-rtd-theme (==0.4.3)", "sphinxcontrib-serializinghtml (==1.1.4)", "sphinxcontrib-websupport (==1.2.4)", "urllib3 (==1.26.4)"] +qa = ["flake8 (==5.0.4)", "mypy (==0.971)", "types-setuptools (==67.2.0.1)"] +testing = ["Django", "attrs", "colorama", "docopt", "pytest (<9.0.0)"] + [[package]] name = "jeepney" version = "0.8.0" description = "Low-level, pure Python DBus protocol wrapper." -optional = false +optional = true python-versions = ">=3.7" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and sys_platform == \"linux\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\" and sys_platform == \"linux\"" files = [ {file = "jeepney-0.8.0-py3-none-any.whl", hash = "sha256:c0a454ad016ca575060802ee4d590dd912e35c122fa04e70306de3d076cce755"}, {file = "jeepney-0.8.0.tar.gz", hash = "sha256:5efe48d255973902f6badc3ce55e2aa6c5c3b3bc642059ef3a91247bcfcc5806"}, @@ -1958,10 +2215,10 @@ i18n = ["Babel (>=2.7)"] name = "json-log-formatter" version = "1.1" description = "JSON log formatter" -optional = false +optional = true python-versions = ">=3.6" -groups = ["unstable"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "json_log_formatter-1.1.tar.gz", hash = "sha256:fe8fd801c58c1234df86211720921f60149105ef8d1e2a72966bb61da9bed584"}, ] @@ -1970,10 +2227,10 @@ files = [ name = "jsonschema" version = "4.23.0" description = "An implementation of JSON Schema validation for Python" -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "jsonschema-4.23.0-py3-none-any.whl", hash = "sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566"}, {file = "jsonschema-4.23.0.tar.gz", hash = "sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4"}, @@ -1993,10 +2250,10 @@ format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339- name = "jsonschema-specifications" version = "2024.10.1" description = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" -optional = false +optional = true python-versions = ">=3.9" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "jsonschema_specifications-2024.10.1-py3-none-any.whl", hash = "sha256:a09a0680616357d9a0ecf05c12ad234479f549239d0f5b55f3deea67475da9bf"}, {file = "jsonschema_specifications-2024.10.1.tar.gz", hash = "sha256:0f38b83639958ce1152d02a7f062902c41c8fd20d558b0c34344292d417ae272"}, @@ -2009,10 +2266,10 @@ referencing = ">=0.31.0" name = "keyring" version = "25.5.0" description = "Store and access your passwords safely." -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "keyring-25.5.0-py3-none-any.whl", hash = "sha256:e67f8ac32b04be4714b42fe84ce7dad9c40985b9ca827c592cc303e7c26d9741"}, {file = "keyring-25.5.0.tar.gz", hash = "sha256:4c753b3ec91717fe713c4edd522d625889d8973a349b0e582622f49766de58e6"}, @@ -2040,10 +2297,10 @@ type = ["pygobject-stubs", "pytest-mypy", "shtab", "types-pywin32"] name = "kiwisolver" version = "1.4.7" description = "A fast implementation of the Cassowary constraint solver" -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "kiwisolver-1.4.7-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8a9c83f75223d5e48b0bc9cb1bf2776cf01563e00ade8775ffe13b0b6e1af3a6"}, {file = "kiwisolver-1.4.7-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:58370b1ffbd35407444d57057b57da5d6549d2d854fa30249771775c63b5fe17"}, @@ -2237,10 +2494,10 @@ files = [ name = "lxml" version = "5.3.0" description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." -optional = false +optional = true python-versions = ">=3.6" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "lxml-5.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:dd36439be765e2dde7660212b5275641edbc813e7b24668831a5c8ac91180656"}, {file = "lxml-5.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ae5fe5c4b525aa82b8076c1a59d642c17b6e8739ecf852522c6321852178119d"}, @@ -2393,10 +2650,10 @@ source = ["Cython (>=3.0.11)"] name = "maclookup" version = "1.0.3" description = "Python client library for macaddress.io API." -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "maclookup-1.0.3-py2.py3-none-any.whl", hash = "sha256:33bf8eaebe3b1e4ab4ae9277dd93c78024e0ebf6b3c42f76c37695bc26ce287a"}, {file = "maclookup-1.0.3.tar.gz", hash = "sha256:795e792cd3e03c9bdad77e52904d43ff71d3ac03b360443f99d4bae08a6bffef"}, @@ -2413,10 +2670,10 @@ dev = ["mock", "tox"] name = "maec" version = "4.1.0.17" description = "An API for parsing and creating MAEC content." -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "maec-4.1.0.17-py2.py3-none-any.whl", hash = "sha256:e6566684e606749ff75ef03f7c3454c6fff8f350fa159b4ef17cdd0e5c632ae6"}, {file = "maec-4.1.0.17.tar.gz", hash = "sha256:d163626a11f27e046c3013d313e5b53ba97f8a9f2f563ac523be55dda6420235"}, @@ -2563,10 +2820,10 @@ files = [ name = "matplotlib" version = "3.9.2" description = "Python plotting package" -optional = false +optional = true python-versions = ">=3.9" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "matplotlib-3.9.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:9d78bbc0cbc891ad55b4f39a48c22182e9bdaea7fc0e5dbd364f49f729ca1bbb"}, {file = "matplotlib-3.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c375cc72229614632c87355366bdf2570c2dac01ac66b8ad048d2dabadf2d0d4"}, @@ -2625,14 +2882,30 @@ python-dateutil = ">=2.7" [package.extras] dev = ["meson-python (>=0.13.1)", "numpy (>=1.25)", "pybind11 (>=2.6)", "setuptools (>=64)", "setuptools_scm (>=7)"] +[[package]] +name = "matplotlib-inline" +version = "0.1.7" +description = "Inline Matplotlib backend for Jupyter" +optional = false +python-versions = ">=3.8" +groups = ["test"] +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +files = [ + {file = "matplotlib_inline-0.1.7-py3-none-any.whl", hash = "sha256:df192d39a4ff8f21b1895d72e6a13f5fcc5099f00fa84384e0ea28c2cc0653ca"}, + {file = "matplotlib_inline-0.1.7.tar.gz", hash = "sha256:8423b23ec666be3d16e16b60bdd8ac4e86e840ebd1dd11a30b9f117f2fa0ab90"}, +] + +[package.dependencies] +traitlets = "*" + [[package]] name = "mattermostdriver" version = "7.3.2" description = "A Python Mattermost Driver" -optional = false +optional = true python-versions = ">=3.5" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "mattermostdriver-7.3.2-py3-none-any.whl", hash = "sha256:8c6f15da34873b6c88da8fa8da0342f94bef77fcd16294befd92fea7e008cd97"}, {file = "mattermostdriver-7.3.2.tar.gz", hash = "sha256:2e4d7b4a17d3013e279c6f993746ea18cd60b45d8fa3be24f47bc2de22b9b3b4"}, @@ -2646,10 +2919,10 @@ websockets = ">=8" name = "maxminddb" version = "2.6.2" description = "Reader for the MaxMind DB format" -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "maxminddb-2.6.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7cfdf5c29a2739610700b9fea7f8d68ce81dcf30bb8016f1a1853ef889a2624b"}, {file = "maxminddb-2.6.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:05e873eb82281cef6e787bd40bd1d58b2e496a21b3689346f0d0420988b3cbb1"}, @@ -2752,10 +3025,10 @@ files = [ name = "misp-lib-stix2" version = "3.0.1.2" description = "Produce and consume STIX 2 JSON content" -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "misp_lib_stix2-3.0.1.2-py2.py3-none-any.whl", hash = "sha256:6172a1b540e78129ff0b845a43bd3483e0c5f928a8d1ba370cb552eaab721841"}, {file = "misp_lib_stix2-3.0.1.2.tar.gz", hash = "sha256:9da014b096fae6c2d9d96dcaac1672658fe7bca963ade04c3c9bde5684a5266f"}, @@ -2775,10 +3048,10 @@ taxii = ["taxii2-client (>=2.3.0)"] name = "misp-stix" version = "2025.1.10" description = "Python scripts used by MISP to export MISP format into STIX and to import STIX into MISP format." -optional = false +optional = true python-versions = "<4.0,>=3.9" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "misp_stix-2025.1.10-py3-none-any.whl", hash = "sha256:76a853435aedae7ed448ec0bf1b0dc65cfbb6241cd4dec9ba6ccd3ec8a412576"}, {file = "misp_stix-2025.1.10.tar.gz", hash = "sha256:12180e27fc795d5e208f6e9c682eb406477aae1685fa9e7d5dee98e8dbecc1b0"}, @@ -2798,10 +3071,10 @@ stix-edh = "1.0.3" name = "mixbox" version = "1.0.5" description = "Utility library for cybox, maec, and stix packages" -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "mixbox-1.0.5-py2.py3-none-any.whl", hash = "sha256:170551ec415a0705b3c26c92e3f75fe1cdc95cc7f71f2b6ada4248fb1e96d035"}, {file = "mixbox-1.0.5.tar.gz", hash = "sha256:13c618a36967a6906d09e9e5be952656c78279b0e9cb5527e9360416e4d1c057"}, @@ -2913,10 +3186,10 @@ files = [ name = "more-itertools" version = "10.5.0" description = "More routines for operating on iterables, beyond itertools" -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "more-itertools-10.5.0.tar.gz", hash = "sha256:5482bfef7849c25dc3c6dd53a6173ae4795da2a41a80faea6700d9f5846c5da6"}, {file = "more_itertools-10.5.0-py3-none-any.whl", hash = "sha256:037b0d3203ce90cca8ab1defbbdac29d5f993fc20131f3664dc8d6acfa872aef"}, @@ -2929,7 +3202,7 @@ description = "Python tool and library for decrypting and encrypting MS Office f optional = false python-versions = "<4.0,>=3.8" groups = ["main"] -markers = "platform_python_implementation != \"PyPy\" or platform_system == \"Linux\" and platform_machine == \"aarch64\" or platform_system != \"Windows\" and platform_system != \"Darwin\"" +markers = "platform_python_implementation != \"PyPy\" or platform_system != \"Windows\" and platform_system != \"Darwin\"" files = [ {file = "msoffcrypto_tool-5.4.2-py3-none-any.whl", hash = "sha256:274fe2181702d1e5a107ec1b68a4c9fea997a44972ae1cc9ae0cb4f6a50fef0e"}, {file = "msoffcrypto_tool-5.4.2.tar.gz", hash = "sha256:44b545adba0407564a0cc3d6dde6ca36b7c0fdf352b85bca51618fa1d4817370"}, @@ -2943,10 +3216,10 @@ olefile = ">=0.46" name = "multidict" version = "6.1.0" description = "multidict implementation" -optional = false +optional = true python-versions = ">=3.8" -groups = ["main", "unstable"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "multidict-6.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3380252550e372e8511d49481bd836264c009adb826b23fefcc5dd3c69692f60"}, {file = "multidict-6.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:99f826cbf970077383d7de805c0681799491cb939c25450b9b5b3ced03ca99f1"}, @@ -3049,10 +3322,10 @@ typing-extensions = {version = ">=4.1.0", markers = "python_version < \"3.11\""} name = "mwdblib" version = "4.5.0" description = "MWDB API bindings for Python" -optional = false +optional = true python-versions = ">=3.7" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "mwdblib-4.5.0-py3-none-any.whl", hash = "sha256:94fc48ad92a3cd44badc894ed5f4c70fd3ce4fbdd3a567f499ae85db9c5f7048"}, ] @@ -3064,14 +3337,27 @@ requests = "*" [package.extras] cli = ["beautifultable (>=1.0.0)", "click (>=7.0)", "click-default-group", "humanize (>=0.5.1)"] +[[package]] +name = "mypy-extensions" +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." +optional = false +python-versions = ">=3.5" +groups = ["test"] +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] + [[package]] name = "ndjson" version = "0.3.1" description = "JsonDecoder for ndjson" -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "ndjson-0.3.1-py2.py3-none-any.whl", hash = "sha256:839c22275e6baa3040077b83c005ac24199b94973309a8a1809be962c753a410"}, {file = "ndjson-0.3.1.tar.gz", hash = "sha256:bf9746cb6bb1cb53d172cda7f154c07c786d665ff28341e4e689b796b229e5d6"}, @@ -3095,10 +3381,10 @@ files = [ name = "np" version = "1.0.2" description = "np = numpy++: numpy with added convenience functionality" -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "np-1.0.2.tar.gz", hash = "sha256:781265283f3823663ad8fb48741aae62abcf4c78bc19f908f8aa7c1d3eb132f8"}, ] @@ -3107,10 +3393,10 @@ files = [ name = "numpy" version = "1.26.4" description = "Fundamental package for array computing in Python" -optional = false +optional = true python-versions = ">=3.9" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0"}, {file = "numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a"}, @@ -3154,10 +3440,10 @@ files = [ name = "oauth2" version = "1.9.0.post1" description = "library for OAuth version 1.9" -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "oauth2-1.9.0.post1-py2.py3-none-any.whl", hash = "sha256:15b5c42301f46dd63113f1214b0d81a8b16254f65a86d3c32a1b52297f3266e6"}, {file = "oauth2-1.9.0.post1.tar.gz", hash = "sha256:c006a85e7c60107c7cc6da1b184b5c719f6dd7202098196dfa6e55df669b59bf"}, @@ -3166,23 +3452,6 @@ files = [ [package.dependencies] httplib2 = "*" -[[package]] -name = "ODTReader" -version = "0.0.3" -description = "Lightweight python module to allow extracting text from OpenDocument (odt) files." -optional = false -python-versions = "*" -groups = ["unstable"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" -files = [] -develop = false - -[package.source] -type = "git" -url = "https://github.com/cartertemm/ODTReader.git" -reference = "HEAD" -resolved_reference = "49d6938693f6faa3ff09998f86dba551ae3a996b" - [[package]] name = "olefile" version = "0.47" @@ -3227,10 +3496,10 @@ full = ["XLMMacroDeobfuscator"] name = "opencv-python" version = "4.10.0.84" description = "Wrapper package for OpenCV python bindings." -optional = false +optional = true python-versions = ">=3.6" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "opencv-python-4.10.0.84.tar.gz", hash = "sha256:72d234e4582e9658ffea8e9cae5b63d488ad06994ef12d81dc303b17472f3526"}, {file = "opencv_python-4.10.0.84-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:fc182f8f4cda51b45f01c64e4cbedfc2f00aff799debebc305d8d0210c43f251"}, @@ -3255,10 +3524,10 @@ numpy = [ name = "openpyxl" version = "3.1.5" description = "A Python library to read/write Excel 2010 xlsx/xlsm files" -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2"}, {file = "openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050"}, @@ -3271,10 +3540,10 @@ et-xmlfile = "*" name = "ordered-set" version = "4.1.0" description = "An OrderedSet is a custom MutableSet that remembers its order, so that every" -optional = false +optional = true python-versions = ">=3.7" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "ordered-set-4.1.0.tar.gz", hash = "sha256:694a8e44c87657c59292ede72891eb91d34131f6531463aab3009191c77364a8"}, {file = "ordered_set-4.1.0-py3-none-any.whl", hash = "sha256:046e1132c71fcf3330438a539928932caf51ddbc582496833e23de611de14562"}, @@ -3283,6 +3552,96 @@ files = [ [package.extras] dev = ["black", "mypy", "pytest"] +[[package]] +name = "orjson" +version = "3.10.15" +description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" +optional = false +python-versions = ">=3.8" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +files = [ + {file = "orjson-3.10.15-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:552c883d03ad185f720d0c09583ebde257e41b9521b74ff40e08b7dec4559c04"}, + {file = "orjson-3.10.15-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:616e3e8d438d02e4854f70bfdc03a6bcdb697358dbaa6bcd19cbe24d24ece1f8"}, + {file = "orjson-3.10.15-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7c2c79fa308e6edb0ffab0a31fd75a7841bf2a79a20ef08a3c6e3b26814c8ca8"}, + {file = "orjson-3.10.15-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:73cb85490aa6bf98abd20607ab5c8324c0acb48d6da7863a51be48505646c814"}, + {file = "orjson-3.10.15-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:763dadac05e4e9d2bc14938a45a2d0560549561287d41c465d3c58aec818b164"}, + {file = "orjson-3.10.15-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a330b9b4734f09a623f74a7490db713695e13b67c959713b78369f26b3dee6bf"}, + {file = "orjson-3.10.15-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a61a4622b7ff861f019974f73d8165be1bd9a0855e1cad18ee167acacabeb061"}, + {file = "orjson-3.10.15-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acd271247691574416b3228db667b84775c497b245fa275c6ab90dc1ffbbd2b3"}, + {file = "orjson-3.10.15-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:e4759b109c37f635aa5c5cc93a1b26927bfde24b254bcc0e1149a9fada253d2d"}, + {file = "orjson-3.10.15-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:9e992fd5cfb8b9f00bfad2fd7a05a4299db2bbe92e6440d9dd2fab27655b3182"}, + {file = "orjson-3.10.15-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f95fb363d79366af56c3f26b71df40b9a583b07bbaaf5b317407c4d58497852e"}, + {file = "orjson-3.10.15-cp310-cp310-win32.whl", hash = "sha256:f9875f5fea7492da8ec2444839dcc439b0ef298978f311103d0b7dfd775898ab"}, + {file = "orjson-3.10.15-cp310-cp310-win_amd64.whl", hash = "sha256:17085a6aa91e1cd70ca8533989a18b5433e15d29c574582f76f821737c8d5806"}, + {file = "orjson-3.10.15-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:c4cc83960ab79a4031f3119cc4b1a1c627a3dc09df125b27c4201dff2af7eaa6"}, + {file = "orjson-3.10.15-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ddbeef2481d895ab8be5185f2432c334d6dec1f5d1933a9c83014d188e102cef"}, + {file = "orjson-3.10.15-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9e590a0477b23ecd5b0ac865b1b907b01b3c5535f5e8a8f6ab0e503efb896334"}, + {file = "orjson-3.10.15-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a6be38bd103d2fd9bdfa31c2720b23b5d47c6796bcb1d1b598e3924441b4298d"}, + {file = "orjson-3.10.15-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ff4f6edb1578960ed628a3b998fa54d78d9bb3e2eb2cfc5c2a09732431c678d0"}, + {file = "orjson-3.10.15-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b0482b21d0462eddd67e7fce10b89e0b6ac56570424662b685a0d6fccf581e13"}, + {file = "orjson-3.10.15-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bb5cc3527036ae3d98b65e37b7986a918955f85332c1ee07f9d3f82f3a6899b5"}, + {file = "orjson-3.10.15-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d569c1c462912acdd119ccbf719cf7102ea2c67dd03b99edcb1a3048651ac96b"}, + {file = "orjson-3.10.15-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:1e6d33efab6b71d67f22bf2962895d3dc6f82a6273a965fab762e64fa90dc399"}, + {file = "orjson-3.10.15-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:c33be3795e299f565681d69852ac8c1bc5c84863c0b0030b2b3468843be90388"}, + {file = "orjson-3.10.15-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:eea80037b9fae5339b214f59308ef0589fc06dc870578b7cce6d71eb2096764c"}, + {file = "orjson-3.10.15-cp311-cp311-win32.whl", hash = "sha256:d5ac11b659fd798228a7adba3e37c010e0152b78b1982897020a8e019a94882e"}, + {file = "orjson-3.10.15-cp311-cp311-win_amd64.whl", hash = "sha256:cf45e0214c593660339ef63e875f32ddd5aa3b4adc15e662cdb80dc49e194f8e"}, + {file = "orjson-3.10.15-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9d11c0714fc85bfcf36ada1179400862da3288fc785c30e8297844c867d7505a"}, + {file = "orjson-3.10.15-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dba5a1e85d554e3897fa9fe6fbcff2ed32d55008973ec9a2b992bd9a65d2352d"}, + {file = "orjson-3.10.15-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7723ad949a0ea502df656948ddd8b392780a5beaa4c3b5f97e525191b102fff0"}, + {file = "orjson-3.10.15-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6fd9bc64421e9fe9bd88039e7ce8e58d4fead67ca88e3a4014b143cec7684fd4"}, + {file = "orjson-3.10.15-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dadba0e7b6594216c214ef7894c4bd5f08d7c0135f4dd0145600be4fbcc16767"}, + {file = "orjson-3.10.15-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b48f59114fe318f33bbaee8ebeda696d8ccc94c9e90bc27dbe72153094e26f41"}, + {file = "orjson-3.10.15-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:035fb83585e0f15e076759b6fedaf0abb460d1765b6a36f48018a52858443514"}, + {file = "orjson-3.10.15-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d13b7fe322d75bf84464b075eafd8e7dd9eae05649aa2a5354cfa32f43c59f17"}, + {file = "orjson-3.10.15-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7066b74f9f259849629e0d04db6609db4cf5b973248f455ba5d3bd58a4daaa5b"}, + {file = "orjson-3.10.15-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:88dc3f65a026bd3175eb157fea994fca6ac7c4c8579fc5a86fc2114ad05705b7"}, + {file = "orjson-3.10.15-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b342567e5465bd99faa559507fe45e33fc76b9fb868a63f1642c6bc0735ad02a"}, + {file = "orjson-3.10.15-cp312-cp312-win32.whl", hash = "sha256:0a4f27ea5617828e6b58922fdbec67b0aa4bb844e2d363b9244c47fa2180e665"}, + {file = "orjson-3.10.15-cp312-cp312-win_amd64.whl", hash = "sha256:ef5b87e7aa9545ddadd2309efe6824bd3dd64ac101c15dae0f2f597911d46eaa"}, + {file = "orjson-3.10.15-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:bae0e6ec2b7ba6895198cd981b7cca95d1487d0147c8ed751e5632ad16f031a6"}, + {file = "orjson-3.10.15-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f93ce145b2db1252dd86af37d4165b6faa83072b46e3995ecc95d4b2301b725a"}, + {file = "orjson-3.10.15-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7c203f6f969210128af3acae0ef9ea6aab9782939f45f6fe02d05958fe761ef9"}, + {file = "orjson-3.10.15-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8918719572d662e18b8af66aef699d8c21072e54b6c82a3f8f6404c1f5ccd5e0"}, + {file = "orjson-3.10.15-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f71eae9651465dff70aa80db92586ad5b92df46a9373ee55252109bb6b703307"}, + {file = "orjson-3.10.15-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e117eb299a35f2634e25ed120c37c641398826c2f5a3d3cc39f5993b96171b9e"}, + {file = "orjson-3.10.15-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:13242f12d295e83c2955756a574ddd6741c81e5b99f2bef8ed8d53e47a01e4b7"}, + {file = "orjson-3.10.15-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7946922ada8f3e0b7b958cc3eb22cfcf6c0df83d1fe5521b4a100103e3fa84c8"}, + {file = "orjson-3.10.15-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:b7155eb1623347f0f22c38c9abdd738b287e39b9982e1da227503387b81b34ca"}, + {file = "orjson-3.10.15-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:208beedfa807c922da4e81061dafa9c8489c6328934ca2a562efa707e049e561"}, + {file = "orjson-3.10.15-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eca81f83b1b8c07449e1d6ff7074e82e3fd6777e588f1a6632127f286a968825"}, + {file = "orjson-3.10.15-cp313-cp313-win32.whl", hash = "sha256:c03cd6eea1bd3b949d0d007c8d57049aa2b39bd49f58b4b2af571a5d3833d890"}, + {file = "orjson-3.10.15-cp313-cp313-win_amd64.whl", hash = "sha256:fd56a26a04f6ba5fb2045b0acc487a63162a958ed837648c5781e1fe3316cfbf"}, + {file = "orjson-3.10.15-cp38-cp38-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:5e8afd6200e12771467a1a44e5ad780614b86abb4b11862ec54861a82d677746"}, + {file = "orjson-3.10.15-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da9a18c500f19273e9e104cca8c1f0b40a6470bcccfc33afcc088045d0bf5ea6"}, + {file = "orjson-3.10.15-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bb00b7bfbdf5d34a13180e4805d76b4567025da19a197645ca746fc2fb536586"}, + {file = "orjson-3.10.15-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:33aedc3d903378e257047fee506f11e0833146ca3e57a1a1fb0ddb789876c1e1"}, + {file = "orjson-3.10.15-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dd0099ae6aed5eb1fc84c9eb72b95505a3df4267e6962eb93cdd5af03be71c98"}, + {file = "orjson-3.10.15-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7c864a80a2d467d7786274fce0e4f93ef2a7ca4ff31f7fc5634225aaa4e9e98c"}, + {file = "orjson-3.10.15-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c25774c9e88a3e0013d7d1a6c8056926b607a61edd423b50eb5c88fd7f2823ae"}, + {file = "orjson-3.10.15-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:e78c211d0074e783d824ce7bb85bf459f93a233eb67a5b5003498232ddfb0e8a"}, + {file = "orjson-3.10.15-cp38-cp38-musllinux_1_2_armv7l.whl", hash = "sha256:43e17289ffdbbac8f39243916c893d2ae41a2ea1a9cbb060a56a4d75286351ae"}, + {file = "orjson-3.10.15-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:781d54657063f361e89714293c095f506c533582ee40a426cb6489c48a637b81"}, + {file = "orjson-3.10.15-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:6875210307d36c94873f553786a808af2788e362bd0cf4c8e66d976791e7b528"}, + {file = "orjson-3.10.15-cp38-cp38-win32.whl", hash = "sha256:305b38b2b8f8083cc3d618927d7f424349afce5975b316d33075ef0f73576b60"}, + {file = "orjson-3.10.15-cp38-cp38-win_amd64.whl", hash = "sha256:5dd9ef1639878cc3efffed349543cbf9372bdbd79f478615a1c633fe4e4180d1"}, + {file = "orjson-3.10.15-cp39-cp39-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:ffe19f3e8d68111e8644d4f4e267a069ca427926855582ff01fc012496d19969"}, + {file = "orjson-3.10.15-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d433bf32a363823863a96561a555227c18a522a8217a6f9400f00ddc70139ae2"}, + {file = "orjson-3.10.15-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:da03392674f59a95d03fa5fb9fe3a160b0511ad84b7a3914699ea5a1b3a38da2"}, + {file = "orjson-3.10.15-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3a63bb41559b05360ded9132032239e47983a39b151af1201f07ec9370715c82"}, + {file = "orjson-3.10.15-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3766ac4702f8f795ff3fa067968e806b4344af257011858cc3d6d8721588b53f"}, + {file = "orjson-3.10.15-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a1c73dcc8fadbd7c55802d9aa093b36878d34a3b3222c41052ce6b0fc65f8e8"}, + {file = "orjson-3.10.15-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b299383825eafe642cbab34be762ccff9fd3408d72726a6b2a4506d410a71ab3"}, + {file = "orjson-3.10.15-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:abc7abecdbf67a173ef1316036ebbf54ce400ef2300b4e26a7b843bd446c2480"}, + {file = "orjson-3.10.15-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:3614ea508d522a621384c1d6639016a5a2e4f027f3e4a1c93a51867615d28829"}, + {file = "orjson-3.10.15-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:295c70f9dc154307777ba30fe29ff15c1bcc9dfc5c48632f37d20a607e9ba85a"}, + {file = "orjson-3.10.15-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:63309e3ff924c62404923c80b9e2048c1f74ba4b615e7584584389ada50ed428"}, + {file = "orjson-3.10.15-cp39-cp39-win32.whl", hash = "sha256:a2f708c62d026fb5340788ba94a55c23df4e1869fec74be455e0b2f5363b8507"}, + {file = "orjson-3.10.15-cp39-cp39-win_amd64.whl", hash = "sha256:efcf6c735c3d22ef60c4aa27a5238f1a477df85e9b15f2142f9d669beb2d13fd"}, + {file = "orjson-3.10.15.tar.gz", hash = "sha256:05ca7fe452a2e9d8d9d706a2984c95b9c2ebc5db417ce0b7a49b91d50642a23e"}, +] + [[package]] name = "packaging" version = "24.2" @@ -3290,11 +3649,11 @@ description = "Core utilities for Python packages" optional = false python-versions = ">=3.8" groups = ["main", "docs", "test"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" files = [ {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"}, {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"}, ] +markers = {main = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"", docs = "platform_system == \"Linux\" and platform_machine == \"aarch64\"", test = "platform_system == \"Linux\" and platform_machine == \"aarch64\""} [[package]] name = "paginate" @@ -3317,10 +3676,10 @@ lint = ["black"] name = "pandas" version = "2.2.3" description = "Powerful data structures for data analysis, time series, and statistics" -optional = false +optional = true python-versions = ">=3.9" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "pandas-2.2.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1948ddde24197a0f7add2bdc4ca83bf2b1ef84a1bc8ccffd95eda17fd836ecb5"}, {file = "pandas-2.2.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:381175499d3802cde0eabbaf6324cce0c4f5d52ca6f8c377c29ad442f50f6348"}, @@ -3405,10 +3764,10 @@ xml = ["lxml (>=4.9.2)"] name = "pandas-ods-reader" version = "1.0.1" description = "Read in .ods and .fods files and return a pandas.DataFrame." -optional = false +optional = true python-versions = "<3.13,>=3.9" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "pandas_ods_reader-1.0.1-py3-none-any.whl", hash = "sha256:35a254ec95665fb654b573d7131787bf177e9126b06746ea3c17b1b972abc79b"}, {file = "pandas_ods_reader-1.0.1.tar.gz", hash = "sha256:e87806d72bba31845de9f0dfa6c5621a5aa9b120cb84049544fee0e8baad8f9c"}, @@ -3423,10 +3782,10 @@ pandas = ">=1.5.2" name = "pandoc" version = "2.4" description = "Pandoc Documents for Python" -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "pandoc-2.4.tar.gz", hash = "sha256:ecd1f8cbb7f4180c6b5db4a17a7c1a74df519995f5f186ef81ce72a9cbd0dd9a"}, ] @@ -3435,14 +3794,31 @@ files = [ plumbum = "*" ply = "*" +[[package]] +name = "parso" +version = "0.8.4" +description = "A Python Parser" +optional = false +python-versions = ">=3.6" +groups = ["test"] +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +files = [ + {file = "parso-0.8.4-py2.py3-none-any.whl", hash = "sha256:a418670a20291dacd2dddc80c377c5c3791378ee1e8d12bffc35420643d43f18"}, + {file = "parso-0.8.4.tar.gz", hash = "sha256:eb3a7b58240fb99099a345571deecc0f9540ea5f4dd2fe14c2a99d6b281ab92d"}, +] + +[package.extras] +qa = ["flake8 (==5.0.4)", "mypy (==0.971)", "types-setuptools (==67.2.0.1)"] +testing = ["docopt", "pytest"] + [[package]] name = "passivetotal" version = "2.5.9" description = "Library for the RiskIQ PassiveTotal and Illuminate API" -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "passivetotal-2.5.9-py3-none-any.whl", hash = "sha256:070c408181bf294f1cf4d49bd7184a00c9419b2bac7a3405f247f786db45ed8f"}, {file = "passivetotal-2.5.9.tar.gz", hash = "sha256:f5f1b7843257bc1ed5ae951c48902eb809a4a632947a57d6f8ad199428b13251"}, @@ -3463,7 +3839,7 @@ version = "0.12.1" description = "Utility library for gitignore style pattern matching of file paths." optional = false python-versions = ">=3.8" -groups = ["docs"] +groups = ["docs", "test"] markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" files = [ {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, @@ -3491,14 +3867,30 @@ win-unicode-console = {version = "*", markers = "platform_system == \"Windows\" name = "pdftotext" version = "2.2.2" description = "Simple PDF text extraction" -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "pdftotext-2.2.2.tar.gz", hash = "sha256:2a9aa89bc62022408781b39d188fabf5a3ad1103b6630f32c4e27e395f7966ee"}, ] +[[package]] +name = "pexpect" +version = "4.9.0" +description = "Pexpect allows easy control of interactive console applications." +optional = false +python-versions = "*" +groups = ["test"] +markers = "python_version == \"3.9\" and sys_platform != \"win32\" or platform_system == \"Linux\" and platform_machine == \"aarch64\" and python_version < \"3.10\" and sys_platform != \"win32\" or sys_platform != \"win32\" and sys_platform != \"emscripten\"" +files = [ + {file = "pexpect-4.9.0-py2.py3-none-any.whl", hash = "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523"}, + {file = "pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f"}, +] + +[package.dependencies] +ptyprocess = ">=0.5" + [[package]] name = "pillow" version = "11.0.0" @@ -3599,12 +3991,12 @@ version = "4.3.6" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.8" -groups = ["main", "docs"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +groups = ["main", "docs", "test"] files = [ {file = "platformdirs-4.3.6-py3-none-any.whl", hash = "sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb"}, {file = "platformdirs-4.3.6.tar.gz", hash = "sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907"}, ] +markers = {main = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"", docs = "platform_system == \"Linux\" and platform_machine == \"aarch64\"", test = "platform_system == \"Linux\" and platform_machine == \"aarch64\""} [package.extras] docs = ["furo (>=2024.8.6)", "proselint (>=0.14)", "sphinx (>=8.0.2)", "sphinx-autodoc-typehints (>=2.4)"] @@ -3632,10 +4024,10 @@ testing = ["pytest", "pytest-benchmark"] name = "plumbum" version = "1.9.0" description = "Plumbum: shell combinators library" -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "plumbum-1.9.0-py3-none-any.whl", hash = "sha256:9fd0d3b0e8d86e4b581af36edf3f3bbe9d1ae15b45b8caab28de1bcb27aaa7f5"}, {file = "plumbum-1.9.0.tar.gz", hash = "sha256:e640062b72642c3873bd5bdc3effed75ba4d3c70ef6b6a7b907357a84d909219"}, @@ -3654,10 +4046,10 @@ test = ["coverage[toml]", "paramiko", "psutil", "pytest (>=6.0)", "pytest-cov", name = "ply" version = "3.11" description = "Python Lex & Yacc" -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "ply-3.11-py2.py3-none-any.whl", hash = "sha256:096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce"}, {file = "ply-3.11.tar.gz", hash = "sha256:00c7c1aaa88358b9c765b6d3000c6eec0ba42abca5351b095321aef446081da3"}, @@ -3667,10 +4059,10 @@ files = [ name = "progressbar2" version = "4.5.0" description = "A Python Progressbar library to provide visual (yet text based) progress to long running operations." -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "progressbar2-4.5.0-py3-none-any.whl", hash = "sha256:625c94a54e63915b3959355e6d4aacd63a00219e5f3e2b12181b76867bf6f628"}, {file = "progressbar2-4.5.0.tar.gz", hash = "sha256:6662cb624886ed31eb94daf61e27583b5144ebc7383a17bae076f8f4f59088fb"}, @@ -3689,12 +4081,12 @@ version = "3.0.48" description = "Library for building powerful interactive command lines in Python" optional = false python-versions = ">=3.7.0" -groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +groups = ["main", "test"] files = [ {file = "prompt_toolkit-3.0.48-py3-none-any.whl", hash = "sha256:f49a827f90062e411f1ce1f854f2aedb3c23353244f8108b89283587397ac10e"}, {file = "prompt_toolkit-3.0.48.tar.gz", hash = "sha256:d6623ab0477a80df74e646bdbc93621143f5caf104206aa29294d53de1a03d90"}, ] +markers = {main = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"", test = "platform_system == \"Linux\" and platform_machine == \"aarch64\""} [package.dependencies] wcwidth = "*" @@ -3703,10 +4095,10 @@ wcwidth = "*" name = "propcache" version = "0.2.0" description = "Accelerated property cache" -optional = false +optional = true python-versions = ">=3.8" -groups = ["main", "unstable"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "propcache-0.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c5869b8fd70b81835a6f187c5fdbe67917a04d7e52b6e7cc4e5fe39d55c39d58"}, {file = "propcache-0.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:952e0d9d07609d9c5be361f33b0d6d650cd2bae393aabb11d9b719364521984b"}, @@ -3840,6 +4232,19 @@ files = [ dev = ["black", "check-manifest", "coverage", "packaging", "pylint", "pyperf", "pypinfo", "pytest-cov", "requests", "rstcheck", "ruff", "sphinx", "sphinx_rtd_theme", "toml-sort", "twine", "virtualenv", "wheel"] test = ["pytest", "pytest-xdist", "setuptools"] +[[package]] +name = "ptyprocess" +version = "0.7.0" +description = "Run a subprocess in a pseudo terminal" +optional = false +python-versions = "*" +groups = ["test"] +markers = "python_version == \"3.9\" and sys_platform != \"win32\" or platform_system == \"Linux\" and platform_machine == \"aarch64\" and python_version < \"3.10\" and sys_platform != \"win32\" or sys_platform != \"win32\" and sys_platform != \"emscripten\"" +files = [ + {file = "ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35"}, + {file = "ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220"}, +] + [[package]] name = "publicsuffixlist" version = "1.0.2.20250212" @@ -3857,6 +4262,22 @@ files = [ readme = ["pandoc"] update = ["requests"] +[[package]] +name = "pure-eval" +version = "0.2.3" +description = "Safely evaluate AST nodes without side effects" +optional = false +python-versions = "*" +groups = ["test"] +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +files = [ + {file = "pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0"}, + {file = "pure_eval-0.2.3.tar.gz", hash = "sha256:5f4e983f40564c576c7c8635ae88db5956bb2229d7e9237d03b3c0b0190eaf42"}, +] + +[package.extras] +tests = ["pytest"] + [[package]] name = "pycodestyle" version = "2.12.1" @@ -3874,10 +4295,10 @@ files = [ name = "pycountry" version = "24.6.1" description = "ISO country, subdivision, language, currency and script definitions and their translations" -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "pycountry-24.6.1-py3-none-any.whl", hash = "sha256:f1a4fb391cd7214f8eefd39556d740adcc233c778a27f8942c8dca351d6ce06f"}, {file = "pycountry-24.6.1.tar.gz", hash = "sha256:b61b3faccea67f87d10c1f2b0fc0be714409e8fcdcc1315613174f6466c10221"}, @@ -3900,10 +4321,10 @@ files = [ name = "pycryptodome" version = "3.21.0" description = "Cryptographic library for Python" -optional = false +optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "pycryptodome-3.21.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:dad9bf36eda068e89059d1f07408e397856be9511d7113ea4b586642a429a4fd"}, {file = "pycryptodome-3.21.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:a1752eca64c60852f38bb29e2c86fca30d7672c024128ef5d70cc15868fa10f4"}, @@ -3943,10 +4364,10 @@ files = [ name = "pycryptodomex" version = "3.21.0" description = "Cryptographic library for Python" -optional = false +optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "extra == \"all\"" files = [ {file = "pycryptodomex-3.21.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:dbeb84a399373df84a69e0919c1d733b89e049752426041deeb30d68e9867822"}, {file = "pycryptodomex-3.21.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:a192fb46c95489beba9c3f002ed7d93979423d1b2a53eab8771dbb1339eb3ddd"}, @@ -3986,10 +4407,10 @@ files = [ name = "pydantic" version = "2.10.1" description = "Data validation using Python type hints" -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "pydantic-2.10.1-py3-none-any.whl", hash = "sha256:a8d20db84de64cf4a7d59e899c2caf0fe9d660c7cfc482528e7020d7dd189a7e"}, {file = "pydantic-2.10.1.tar.gz", hash = "sha256:a4daca2dc0aa429555e0656d6bf94873a7dc5f54ee42b1f5873d666fb3f35560"}, @@ -4008,10 +4429,10 @@ timezone = ["tzdata ; python_version >= \"3.9\" and platform_system == \"Windows name = "pydantic-core" version = "2.27.1" description = "Core functionality for Pydantic validation and serialization" -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "pydantic_core-2.27.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:71a5e35c75c021aaf400ac048dacc855f000bdfed91614b4a726f7432f1f3d6a"}, {file = "pydantic_core-2.27.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f82d068a2d6ecfc6e054726080af69a6764a10015467d7d7b9f66d6ed5afa23b"}, @@ -4144,31 +4565,14 @@ files = [ {file = "pydeep2-0.5.1.tar.gz", hash = "sha256:44ce447e3253a69d3393f3cc53e3a87a48fe3ff9861793736a7bc218a1b95d77"}, ] -[[package]] -name = "pydnstrails" -version = "1.0" -description = "" -optional = false -python-versions = "*" -groups = ["unstable"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" -files = [] -develop = false - -[package.source] -type = "git" -url = "https://github.com/sebdraven/pydnstrails.git" -reference = "HEAD" -resolved_reference = "48c1f740025c51289f43a24863d1845ff12fd21a" - [[package]] name = "pyeti-python3" version = "1.1" description = "Revival version of pyeti, the API for Yeti Threat Intel Platform." -optional = false +optional = true python-versions = ">=3.6" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "pyeti_python3-1.1-py3-none-any.whl", hash = "sha256:2b2011fadacf799bd99bcb5c3feec4ffafc031bb81e1ab713cd977948ca7d698"}, ] @@ -4181,10 +4585,10 @@ tqdm = "*" name = "pyeupi" version = "1.3.0" description = "Python API for the European Union anti-phishing initiative." -optional = false +optional = true python-versions = ">=3.8,<4.0" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "pyeupi-1.3.0-py3-none-any.whl", hash = "sha256:495248912ff3635d1c3e8b9c6e012add117178440fdd6c8d75cb2019990df996"}, {file = "pyeupi-1.3.0.tar.gz", hash = "sha256:a262d8f1c6697c2d9fd462725217e33efa54f0a0c8104ce07fb8a3be781e4d72"}, @@ -4213,10 +4617,10 @@ files = [ name = "pygeoip" version = "0.3.2" description = "Pure Python GeoIP API" -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "pygeoip-0.3.2-py2.py3-none-any.whl", hash = "sha256:1938b9dac7b00d77f94d040b9465ea52c938f3fcdcd318b5537994f3c16aef96"}, {file = "pygeoip-0.3.2.tar.gz", hash = "sha256:f22c4e00ddf1213e0fae36dc60b46ee7c25a6339941ec1a975539014c1f9a96d"}, @@ -4228,12 +4632,12 @@ version = "2.18.0" description = "Pygments is a syntax highlighting package written in Python." optional = false python-versions = ">=3.8" -groups = ["main", "docs"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +groups = ["main", "docs", "test"] files = [ {file = "pygments-2.18.0-py3-none-any.whl", hash = "sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a"}, {file = "pygments-2.18.0.tar.gz", hash = "sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199"}, ] +markers = {main = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"", docs = "platform_system == \"Linux\" and platform_machine == \"aarch64\"", test = "platform_system == \"Linux\" and platform_machine == \"aarch64\""} [package.extras] windows-terminal = ["colorama (>=0.4.6)"] @@ -4242,10 +4646,10 @@ windows-terminal = ["colorama (>=0.4.6)"] name = "pyintel471" version = "0.1.1" description = "Python client for Intel471" -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "pyintel471-0.1.1-py3-none-any.whl", hash = "sha256:4e30246d3a5904d437d3653d8c7d82a4fe74b1973b95665d94275a070cab2231"}, {file = "pyintel471-0.1.1.tar.gz", hash = "sha256:81e20fcc09b27d346977492edea693536d5709da5abae78993beaa3e09baff22"}, @@ -4255,10 +4659,10 @@ files = [ name = "pyipasnhistory" version = "2.1.2" description = "Python client for IP ASN History" -optional = false +optional = true python-versions = ">=3.8,<4.0" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "pyipasnhistory-2.1.2-py3-none-any.whl", hash = "sha256:7743de1bb7e735f9b907a3cff8ab189a1d8b5517b56b64f151fc4793b2863e35"}, {file = "pyipasnhistory-2.1.2.tar.gz", hash = "sha256:10aed86bfbaedc8a119cdd5f59eca646938eb266c717f10394ba9fc2199f0281"}, @@ -4327,23 +4731,6 @@ pdfexport = ["reportlab (>=4.2.5,<5.0.0)"] url = ["pyfaup (>=1.2,<2.0)"] virustotal = ["validators (>=0.34.0,<0.35.0)"] -[[package]] -name = "pyonyphe" -version = "2.0" -description = "" -optional = false -python-versions = "*" -groups = ["unstable"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" -files = [] -develop = false - -[package.source] -type = "git" -url = "https://github.com/sebdraven/pyonyphe.git" -reference = "HEAD" -resolved_reference = "d1d6741f8ea4475f3bb77ff20c876f08839cabd1" - [[package]] name = "pyparsing" version = "3.2.0" @@ -4364,10 +4751,10 @@ diagrams = ["jinja2", "railroad-diagrams"] name = "pypdns" version = "2.2.7" description = "Python API for PDNS." -optional = false +optional = true python-versions = "<4.0,>=3.8" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "pypdns-2.2.7-py3-none-any.whl", hash = "sha256:5a7cdabba5587ca144487a3531a18ce499c456f48ba26a4bf1be747e90dffd7b"}, {file = "pypdns-2.2.7.tar.gz", hash = "sha256:124485c61ea6ed7d9680ebabc77b6ee1ad0b57d515d1249acc61f9ab618b3bdb"}, @@ -4384,10 +4771,10 @@ docs = ["Sphinx (<7.2) ; python_version < \"3.9\"", "Sphinx (>=7.2,<8.0) ; pytho name = "pypssl" version = "2.2" description = "Python API for PSSL." -optional = false +optional = true python-versions = ">=3.6,<4.0" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "pypssl-2.2-py3-none-any.whl", hash = "sha256:88cedaa4191b50154951fce98396521ad6c1d7e3eb914343e7a12ec0df1882a8"}, {file = "pypssl-2.2.tar.gz", hash = "sha256:249ea2152827c10e746fe94c2957c0a525f8ed7ca9db2cd972690a3a136d7bb7"}, @@ -4401,10 +4788,10 @@ requests = ">=2.25.1,<3.0.0" name = "pysafebrowsing" version = "0.1.3" description = "Google Safe Browsing API python wrapper" -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "pysafebrowsing-0.1.3-py3-none-any.whl", hash = "sha256:156d3eb259194e2fa155d6c3c60b2bbba7b8f3235b26f964d91353b40e87b5c5"}, {file = "pysafebrowsing-0.1.3.tar.gz", hash = "sha256:9e8e0b1bc98d12ad3dd00e1c65dcc0a0ef6f38fe5afb361884d52a2d705d2032"}, @@ -4418,10 +4805,10 @@ requests = ">=2.32.3" name = "pytesseract" version = "0.3.13" description = "Python-tesseract is a python wrapper for Google's Tesseract-OCR" -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "pytesseract-0.3.13-py3-none-any.whl", hash = "sha256:7a99c6c2ac598360693d83a416e36e0b33a67638bb9d77fdcac094a3589d4b34"}, {file = "pytesseract-0.3.13.tar.gz", hash = "sha256:4bf5f880c99406f52a3cfc2633e42d9dc67615e69d8a509d74867d3baddb5db9"}, @@ -4459,10 +4846,10 @@ dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments name = "python-baseconv" version = "1.2.2" description = "Convert numbers from base 10 integers to base X strings and back again." -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "python-baseconv-1.2.2.tar.gz", hash = "sha256:0539f8bd0464013b05ad62e0a1673f0ac9086c76b43ebf9f833053527cd9931b"}, ] @@ -4473,7 +4860,7 @@ version = "2.9.0.post0" description = "Extensions to the standard Python datetime module" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -groups = ["main", "docs", "unstable"] +groups = ["main", "docs"] markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" files = [ {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, @@ -4487,10 +4874,10 @@ six = ">=1.5" name = "python-docx" version = "1.1.2" description = "Create, read, and update Microsoft Word .docx files." -optional = false +optional = true python-versions = ">=3.7" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "python_docx-1.1.2-py3-none-any.whl", hash = "sha256:08c20d6058916fb19853fcf080f7f42b6270d89eac9fa5f8c15f691c0017fabe"}, {file = "python_docx-1.1.2.tar.gz", hash = "sha256:0cf1f22e95b9002addca7948e16f2cd7acdfd498047f1941ca5d293db7762efd"}, @@ -4504,10 +4891,10 @@ typing-extensions = ">=4.9.0" name = "python-engineio" version = "4.10.1" description = "Engine.IO server and client for Python" -optional = false +optional = true python-versions = ">=3.6" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "python_engineio-4.10.1-py3-none-any.whl", hash = "sha256:445a94004ec8034960ab99e7ce4209ec619c6e6b6a12aedcb05abeab924025c0"}, {file = "python_engineio-4.10.1.tar.gz", hash = "sha256:166cea8dd7429638c5c4e3a4895beae95196e860bc6f29ed0b9fe753d1ef2072"}, @@ -4538,10 +4925,10 @@ files = [ name = "python-pptx" version = "1.0.2" description = "Create, read, and update PowerPoint 2007+ (.pptx) files." -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "python_pptx-1.0.2-py3-none-any.whl", hash = "sha256:160838e0b8565a8b1f67947675886e9fea18aa5e795db7ae531606d68e785cba"}, {file = "python_pptx-1.0.2.tar.gz", hash = "sha256:479a8af0eaf0f0d76b6f00b0887732874ad2e3188230315290cd1f9dd9cc7095"}, @@ -4557,10 +4944,10 @@ XlsxWriter = ">=0.5.7" name = "python-socketio" version = "5.11.4" description = "Socket.IO server and client for Python" -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "python_socketio-5.11.4-py3-none-any.whl", hash = "sha256:42efaa3e3e0b166fc72a527488a13caaac2cefc76174252486503bd496284945"}, {file = "python_socketio-5.11.4.tar.gz", hash = "sha256:8b0b8ff2964b2957c865835e936310190639c00310a47d77321a594d1665355e"}, @@ -4581,10 +4968,10 @@ docs = ["sphinx"] name = "python-utils" version = "3.8.2" description = "Python Utils is a module with some convenient utilities not included with the standard Python install" -optional = false +optional = true python-versions = ">3.8.0" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "python-utils-3.8.2.tar.gz", hash = "sha256:c5d161e4ca58ce3f8c540f035e018850b261a41e7cb98f6ccf8e1deb7174a1f1"}, {file = "python_utils-3.8.2-py2.py3-none-any.whl", hash = "sha256:ad0ccdbd6f856d015cace07f74828b9840b5c4072d9e868a7f6a14fd195555a8"}, @@ -4602,10 +4989,10 @@ tests = ["flake8", "loguru", "pytest", "pytest-asyncio", "pytest-cov", "pytest-m name = "pytz" version = "2024.2" description = "World timezone definitions, modern and historical" -optional = false +optional = true python-versions = "*" -groups = ["main", "unstable"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "pytz-2024.2-py2.py3-none-any.whl", hash = "sha256:31c7c1817eb7fae7ca4b8c7ee50c72f93aa2dd863de768e1ef4245d426aa0725"}, {file = "pytz-2024.2.tar.gz", hash = "sha256:2aa355083c50a0f93fa581709deac0c9ad65cca8a9e9beac660adcbd493c798a"}, @@ -4615,10 +5002,10 @@ files = [ name = "pywin32" version = "308" description = "Python for Window Extensions" -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Windows\" and platform_python_implementation != \"PyPy\"" +markers = "platform_system == \"Windows\" and platform_python_implementation != \"PyPy\" and extra == \"all\"" files = [ {file = "pywin32-308-cp310-cp310-win32.whl", hash = "sha256:796ff4426437896550d2981b9c2ac0ffd75238ad9ea2d3bfa67a1abd546d262e"}, {file = "pywin32-308-cp310-cp310-win_amd64.whl", hash = "sha256:4fc888c59b3c0bef905ce7eb7e2106a07712015ea1c8234b703a088d46110e8e"}, @@ -4644,10 +5031,10 @@ files = [ name = "pywin32-ctypes" version = "0.2.3" description = "A (partial) reimplementation of pywin32 using ctypes/cffi" -optional = false +optional = true python-versions = ">=3.6" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and sys_platform == \"win32\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\" and sys_platform == \"win32\"" files = [ {file = "pywin32-ctypes-0.2.3.tar.gz", hash = "sha256:d162dc04946d704503b2edc4d55f3dba5c1d539ead017afa00142c38b9885755"}, {file = "pywin32_ctypes-0.2.3-py3-none-any.whl", hash = "sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8"}, @@ -4659,8 +5046,7 @@ version = "6.0.2" description = "YAML parser and emitter for Python" optional = false python-versions = ">=3.8" -groups = ["main", "docs", "unstable"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +groups = ["main", "docs"] files = [ {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, @@ -4716,6 +5102,7 @@ files = [ {file = "PyYAML-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8"}, {file = "pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e"}, ] +markers = {main = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"", docs = "platform_system == \"Linux\" and platform_machine == \"aarch64\""} [[package]] name = "pyyaml-env-tag" @@ -4737,10 +5124,10 @@ pyyaml = "*" name = "pyzbar" version = "0.1.9" description = "Read one-dimensional barcodes and QR codes from Python 2 and 3." -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "pyzbar-0.1.9-py2.py3-none-any.whl", hash = "sha256:4559628b8192feb25766d954b36a3753baaf5c97c03135aec7e4a026036b475d"}, {file = "pyzbar-0.1.9-py2.py3-none-win32.whl", hash = "sha256:8f4c5264c9c7c6b9f20d01efc52a4eba1ded47d9ba857a94130afe33703eb518"}, @@ -4754,10 +5141,10 @@ scripts = ["Pillow (>=3.2.0)"] name = "pyzipper" version = "0.3.6" description = "AES encryption for zipfile." -optional = false +optional = true python-versions = ">=3.4" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "extra == \"all\"" files = [ {file = "pyzipper-0.3.6-py2.py3-none-any.whl", hash = "sha256:6d097f465bfa47796b1494e12ea65d1478107d38e13bc56f6e58eedc4f6c1a87"}, {file = "pyzipper-0.3.6.tar.gz", hash = "sha256:0adca90a00c36a93fbe49bfa8c5add452bfe4ef85a1b8e3638739dd1c7b26bfc"}, @@ -4770,10 +5157,10 @@ pycryptodomex = "*" name = "rdflib" version = "7.1.1" description = "RDFLib is a Python library for working with RDF, a simple yet powerful language for representing information." -optional = false +optional = true python-versions = "<4.0.0,>=3.8.1" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "rdflib-7.1.1-py3-none-any.whl", hash = "sha256:e590fa9a2c34ba33a667818b5a84be3fb8a4d85868f8038f17912ec84f912a25"}, {file = "rdflib-7.1.1.tar.gz", hash = "sha256:164de86bd3564558802ca983d84f6616a4a1a420c7a17a8152f5016076b2913e"}, @@ -4826,10 +5213,10 @@ ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==23.2.1)", "requests (>=2.31.0)" name = "referencing" version = "0.35.1" description = "JSON Referencing + Python" -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "referencing-0.35.1-py3-none-any.whl", hash = "sha256:eda6d3234d62814d1c64e305c1331c9a3a6132da475ab6382eaa997b21ee75de"}, {file = "referencing-0.35.1.tar.gz", hash = "sha256:25b42124a6c8b632a425174f24087783efb348a6f1e0008e63cd4466fedf703c"}, @@ -4972,7 +5359,7 @@ version = "2.32.3" description = "Python HTTP for Humans." optional = false python-versions = ">=3.8" -groups = ["main", "docs", "test", "unstable"] +groups = ["main", "docs", "test"] markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" files = [ {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, @@ -4993,10 +5380,10 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "requests-cache" version = "1.2.1" description = "A persistent cache for python requests" -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "requests_cache-1.2.1-py3-none-any.whl", hash = "sha256:1285151cddf5331067baa82598afe2d47c7495a1334bfe7a7d329b43e9fd3603"}, {file = "requests_cache-1.2.1.tar.gz", hash = "sha256:68abc986fdc5b8d0911318fbb5f7c80eebcd4d01bfacc6685ecf8876052511d1"}, @@ -5025,10 +5412,10 @@ yaml = ["pyyaml (>=6.0.1)"] name = "requests-file" version = "2.1.0" description = "File transport adapter for Requests" -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "requests_file-2.1.0-py2.py3-none-any.whl", hash = "sha256:cf270de5a4c5874e84599fc5778303d496c10ae5e870bfa378818f35d21bda5c"}, {file = "requests_file-2.1.0.tar.gz", hash = "sha256:0f549a3f3b0699415ac04d167e9cb39bccfb730cb832b4d20be3d9867356e658"}, @@ -5041,10 +5428,10 @@ requests = ">=1.0.0" name = "rich" version = "10.16.2" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" -optional = false +optional = true python-versions = ">=3.6.2,<4.0.0" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "rich-10.16.2-py3-none-any.whl", hash = "sha256:c59d73bd804c90f747c8d7b1d023b88f2a9ac2454224a4aeaf959b21eeb42d03"}, {file = "rich-10.16.2.tar.gz", hash = "sha256:720974689960e06c2efdb54327f8bf0cdbdf4eae4ad73b6c94213cad405c371b"}, @@ -5062,10 +5449,10 @@ jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] name = "rpds-py" version = "0.21.0" description = "Python bindings to Rust's persistent data structures (rpds)" -optional = false +optional = true python-versions = ">=3.9" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "rpds_py-0.21.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:a017f813f24b9df929674d0332a374d40d7f0162b326562daae8066b502d0590"}, {file = "rpds_py-0.21.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:20cc1ed0bcc86d8e1a7e968cce15be45178fd16e2ff656a243145e0b439bd250"}, @@ -5183,10 +5570,10 @@ msg-parse = ["extract-msg (>=0.27,<1.0)"] name = "ruamel-yaml" version = "0.18.6" description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order" -optional = false +optional = true python-versions = ">=3.7" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "ruamel.yaml-0.18.6-py3-none-any.whl", hash = "sha256:57b53ba33def16c4f3d807c0ccbc00f8a6081827e81ba2491691b76882d0c636"}, {file = "ruamel.yaml-0.18.6.tar.gz", hash = "sha256:8b27e6a217e786c6fbe5634d8f3f11bc63e0f80f6a5890f28863d9c45aac311b"}, @@ -5203,10 +5590,10 @@ jinja2 = ["ruamel.yaml.jinja2 (>=0.2)"] name = "ruamel-yaml-clib" version = "0.2.12" description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml" -optional = false +optional = true python-versions = ">=3.9" groups = ["main"] -markers = "platform_python_implementation == \"CPython\" and python_version == \"3.9\" or platform_python_implementation == \"CPython\" and python_version == \"3.12\" or platform_python_implementation == \"CPython\" and python_version == \"3.11\" or platform_python_implementation == \"CPython\" and python_version == \"3.10\" or platform_python_implementation == \"CPython\" and platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and platform_python_implementation == \"CPython\" and extra == \"all\"" files = [ {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:11f891336688faf5156a36293a9c362bdc7c88f03a8a027c2c1d8e0bcde998e5"}, {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:a606ef75a60ecf3d924613892cc603b154178ee25abb3055db5062da811fd969"}, @@ -5255,10 +5642,10 @@ files = [ name = "secretstorage" version = "3.3.3" description = "Python bindings to FreeDesktop.org Secret Service API" -optional = false +optional = true python-versions = ">=3.6" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and sys_platform == \"linux\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\" and sys_platform == \"linux\"" files = [ {file = "SecretStorage-3.3.3-py3-none-any.whl", hash = "sha256:f356e6628222568e3af06f2eba8df495efa13b3b63081dafd4f7d9a7b7bc9f99"}, {file = "SecretStorage-3.3.3.tar.gz", hash = "sha256:2403533ef369eca6d2ba81718576c5e0f564d5cca1b58f73a8b23e7d4eeebd77"}, @@ -5268,19 +5655,6 @@ files = [ cryptography = ">=2.0" jeepney = ">=0.6" -[[package]] -name = "selenium" -version = "2.53.6" -description = "Python bindings for Selenium" -optional = false -python-versions = "*" -groups = ["unstable"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" -files = [ - {file = "selenium-2.53.6-py2.py3-none-any.whl", hash = "sha256:5071f43daa2e698d60d5633ab0a6630cc68a852b360be99144f1c4c1ace2746c"}, - {file = "selenium-2.53.6.tar.gz", hash = "sha256:f507181f13768d73b98dd9647a466ea5758ef5c7f07b62a285d2bd8de9b27016"}, -] - [[package]] name = "setuptools" version = "75.6.0" @@ -5307,10 +5681,10 @@ type = ["importlib_metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.deve name = "shellingham" version = "1.5.4" description = "Tool to Detect Surrounding Shell" -optional = false +optional = true python-versions = ">=3.7" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686"}, {file = "shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de"}, @@ -5320,10 +5694,10 @@ files = [ name = "shodan" version = "1.31.0" description = "Python library and command-line utility for Shodan (https://developer.shodan.io)" -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "shodan-1.31.0.tar.gz", hash = "sha256:c73275386ea02390e196c35c660706a28dd4d537c5a21eb387ab6236fac251f6"}, ] @@ -5340,10 +5714,10 @@ XlsxWriter = "*" name = "sigmatools" version = "0.23.1" description = "Tools for the Generic Signature Format for SIEM Systems" -optional = false +optional = true python-versions = "~=3.8" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "sigmatools-0.23.1.tar.gz", hash = "sha256:3ff0ba97d9d3ea00cabc3020d38ba5e70d0c6fb1271502b590c1e5b49fbd71de"}, ] @@ -5362,10 +5736,10 @@ test = ["attackcti", "coverage", "yamllint"] name = "sigmf" version = "1.2.3" description = "Easily interact with Signal Metadata Format (SigMF) recordings." -optional = false +optional = true python-versions = ">=3.7" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "sigmf-1.2.3.tar.gz", hash = "sha256:14aa9a72edb2169aab122c30e6f7398ade6a61498361740bda1707e558c98fab"}, ] @@ -5382,10 +5756,10 @@ test = ["hypothesis", "pylint", "pytest", "pytest-cov"] name = "simple-websocket" version = "1.1.0" description = "Simple WebSocket server and client for Python" -optional = false +optional = true python-versions = ">=3.6" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "simple_websocket-1.1.0-py3-none-any.whl", hash = "sha256:4af6069630a38ed6c561010f0e11a5bc0d4ca569b36306eb257cd9a192497c8c"}, {file = "simple_websocket-1.1.0.tar.gz", hash = "sha256:7939234e7aa067c534abdab3a9ed933ec9ce4691b0713c78acb195560aa52ae4"}, @@ -5402,10 +5776,10 @@ docs = ["sphinx"] name = "simplejson" version = "3.19.3" description = "Simple, fast, extensible JSON encoder/decoder for Python" -optional = false +optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.5" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "simplejson-3.19.3-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:f39caec26007a2d0efab6b8b1d74873ede9351962707afab622cc2285dd26ed0"}, {file = "simplejson-3.19.3-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:83c87706265ae3028e8460d08b05f30254c569772e859e5ba61fe8af2c883468"}, @@ -5525,7 +5899,7 @@ version = "1.16.0" description = "Python 2 and 3 compatibility utilities" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" -groups = ["main", "docs", "unstable"] +groups = ["main", "docs"] markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" files = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, @@ -5536,10 +5910,10 @@ files = [ name = "slack-sdk" version = "3.33.4" description = "The Slack API Platform SDK for Python" -optional = false +optional = true python-versions = ">=3.6" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "slack_sdk-3.33.4-py2.py3-none-any.whl", hash = "sha256:9f30cb3c9c07b441c49d53fc27f9f1837ad1592a7e9d4ca431f53cdad8826cc6"}, {file = "slack_sdk-3.33.4.tar.gz", hash = "sha256:5e109847f6b6a22d227609226ba4ed936109dc00675bddeb7e0bee502d3ee7e0"}, @@ -5552,10 +5926,10 @@ optional = ["SQLAlchemy (>=1.4,<3)", "aiodns (>1.0)", "aiohttp (>=3.7.3,<4)", "b name = "sniffio" version = "1.3.1" description = "Sniff out which async library your code is running under" -optional = false +optional = true python-versions = ">=3.7" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2"}, {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"}, @@ -5565,10 +5939,10 @@ files = [ name = "socialscan" version = "1.4.2" description = "Open-source intelligence tool for checking email address and username usage on online platforms" -optional = false +optional = true python-versions = ">=3.6" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "socialscan-1.4.2-py3-none-any.whl", hash = "sha256:47f042bb2ab1afb77c2cf2f31e6ab43afa91ff87849a79307cf753dfc7b84f20"}, {file = "socialscan-1.4.2.tar.gz", hash = "sha256:d03eb63177c516b1b8eb1fbca3d25753bb6d68b56e7325a96414b8b319c5daad"}, @@ -5586,10 +5960,10 @@ tests = ["aiohttp (>=3.5.0)", "colorama", "dataclasses ; python_version < \"3.7\ name = "socketio-client" version = "0.5.7.4" description = "A socket.io client library" -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "socketIO-client-0.5.7.4.tar.gz", hash = "sha256:ef2e362a85ef2816fb224d727319c4b743d63b4dd9e1da99c622c9643fc4e2a0"}, ] @@ -5603,10 +5977,10 @@ websocket-client = ">=0.44.0" name = "softenum" version = "1.0.1" description = "" -optional = false +optional = true python-versions = ">=3.7.0,<4.0.0" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "softenum-1.0.1-py3-none-any.whl", hash = "sha256:df6459434d79ed397a28a39c3f7ea2a21b94d7fb3bad8452361a5f01b22793c1"}, {file = "softenum-1.0.1.tar.gz", hash = "sha256:6c7d9c2b49937b1ba637b2cc3c57db1e470ed7ca9457109b4833a5824bbc1476"}, @@ -5618,7 +5992,7 @@ version = "2.6" description = "A modern CSS selector implementation for Beautiful Soup." optional = false python-versions = ">=3.8" -groups = ["main", "unstable"] +groups = ["main"] markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" files = [ {file = "soupsieve-2.6-py3-none-any.whl", hash = "sha256:e72c4ff06e4fb6e4b5a9f0f55fe6e81514581fca1515028625d0f299c602ccc9"}, @@ -5629,10 +6003,10 @@ files = [ name = "sparqlwrapper" version = "2.0.0" description = "SPARQL Endpoint interface to Python" -optional = false +optional = true python-versions = ">=3.7" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "SPARQLWrapper-2.0.0-py3-none-any.whl", hash = "sha256:c99a7204fff676ee28e6acef327dc1ff8451c6f7217dcd8d49e8872f324a8a20"}, {file = "SPARQLWrapper-2.0.0.tar.gz", hash = "sha256:3fed3ebcc77617a4a74d2644b86fd88e0f32e7f7003ac7b2b334c026201731f1"}, @@ -5647,14 +6021,35 @@ docs = ["sphinx (<5)", "sphinx-rtd-theme"] keepalive = ["keepalive (>=0.5)"] pandas = ["pandas (>=1.3.5)"] +[[package]] +name = "stack-data" +version = "0.6.3" +description = "Extract data from python stack frames and tracebacks for informative displays" +optional = false +python-versions = "*" +groups = ["test"] +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +files = [ + {file = "stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695"}, + {file = "stack_data-0.6.3.tar.gz", hash = "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9"}, +] + +[package.dependencies] +asttokens = ">=2.1.0" +executing = ">=1.2.0" +pure-eval = "*" + +[package.extras] +tests = ["cython", "littleutils", "pygments", "pytest", "typeguard"] + [[package]] name = "stix" version = "1.2.0.11" description = "An API for parsing and generating STIX content." -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "stix-1.2.0.11-py2.py3-none-any.whl", hash = "sha256:a3825e34781d491ac3526111f053db85a2be4549e0c8ce56e05e5eeb5f495e53"}, {file = "stix-1.2.0.11.tar.gz", hash = "sha256:b23a1ca70227e17f42cd0a9f109737f321175f6fe97be5cb24fd4d189dbb1601"}, @@ -5670,10 +6065,10 @@ python-dateutil = "*" name = "stix-edh" version = "1.0.3" description = "An EDH marking extension API for python-stix." -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "stix_edh-1.0.3-py2.py3-none-any.whl", hash = "sha256:709483e53bbd8b77b0267f1cef9879d757f200f196149050b0a805e6b0623d9d"}, ] @@ -5686,10 +6081,10 @@ stix = ">=1.1.1.8,<1.2.1.0" name = "stix2" version = "3.0.1" description = "Produce and consume STIX 2 JSON content" -optional = false +optional = true python-versions = ">=3.6" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "stix2-3.0.1-py2.py3-none-any.whl", hash = "sha256:827acf0b5b319c1b857c9db0d54907bb438b2b32312d236c891a305ad49b0ba2"}, {file = "stix2-3.0.1.tar.gz", hash = "sha256:2a2718dc3451c84c709990b2ca220cc39c75ed23e0864d7e8d8190a9365b0cbf"}, @@ -5709,10 +6104,10 @@ taxii = ["taxii2-client (>=2.3.0)"] name = "stix2-patterns" version = "2.0.0" description = "Validate STIX 2 Patterns." -optional = false +optional = true python-versions = ">=3.6" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "stix2-patterns-2.0.0.tar.gz", hash = "sha256:07750c5a5af2c758e9d2aa4dde9d8e04bcd162ac2a9b0b4c4de4481d443efa08"}, {file = "stix2_patterns-2.0.0-py2.py3-none-any.whl", hash = "sha256:ca4d68b2db42ed99794a418388769d2676ca828e9cac0b8629e73cd3f68f6458"}, @@ -5731,10 +6126,10 @@ test = ["coverage", "pytest", "pytest-cov"] name = "tabulate" version = "0.9.0" description = "Pretty-print tabular data" -optional = false +optional = true python-versions = ">=3.7" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f"}, {file = "tabulate-0.9.0.tar.gz", hash = "sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c"}, @@ -5747,10 +6142,10 @@ widechars = ["wcwidth"] name = "tau-clients" version = "0.3.3" description = "Set of clients to interface with various VMware products" -optional = false +optional = true python-versions = ">=3.9" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "tau_clients-0.3.3-py3-none-any.whl", hash = "sha256:49f34c0b586a28cc5627f3b90f5bfe9c1ab59a12e76c745894c4489b03651891"}, {file = "tau_clients-0.3.3.tar.gz", hash = "sha256:d329a1bb6881e687f2596deff685b4b9372fe42929a26c60c1088f5e7b82f741"}, @@ -5768,10 +6163,10 @@ misp = ["pymisp"] name = "taxii2-client" version = "2.3.0" description = "TAXII 2 Client Library" -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "taxii2-client-2.3.0.tar.gz", hash = "sha256:fb3bf895e2eaff3cd08bb7aad75c9d30682ffc00b9f3add77de3a67dc6b895a3"}, {file = "taxii2_client-2.3.0-py2.py3-none-any.whl", hash = "sha256:b4212b8a8bab170cd5dc386ca3ea36bc44b53932f1da30db150abeef00bce7b9"}, @@ -5790,10 +6185,10 @@ test = ["coverage", "pytest", "pytest-cov", "responses", "tox"] name = "termcolor" version = "2.5.0" description = "ANSI color formatting for output in terminal" -optional = false +optional = true python-versions = ">=3.9" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "termcolor-2.5.0-py3-none-any.whl", hash = "sha256:37b17b5fc1e604945c2642c872a3764b5d547a48009871aea3edd3afa180afb8"}, {file = "termcolor-2.5.0.tar.gz", hash = "sha256:998d8d27da6d48442e8e1f016119076b690d962507531df4890fcd2db2ef8a6f"}, @@ -5806,10 +6201,10 @@ tests = ["pytest", "pytest-cov"] name = "tldextract" version = "5.1.3" description = "Accurately separates a URL's subdomain, domain, and public suffix, using the Public Suffix List (PSL). By default, this includes the public ICANN TLDs and their exceptions. You can optionally support the Public Suffix List's private domains as well." -optional = false +optional = true python-versions = ">=3.9" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "tldextract-5.1.3-py3-none-any.whl", hash = "sha256:78de310cc2ca018692de5ddf320f9d6bd7c5cf857d0fd4f2175f0cdf4440ea75"}, {file = "tldextract-5.1.3.tar.gz", hash = "sha256:d43c7284c23f5dc8a42fd0fee2abede2ff74cc622674e4cb07f514ab3330c338"}, @@ -5894,10 +6289,10 @@ files = [ name = "tqdm" version = "4.67.0" description = "Fast, Extensible Progress Meter" -optional = false +optional = true python-versions = ">=3.7" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "tqdm-4.67.0-py3-none-any.whl", hash = "sha256:0cd8af9d56911acab92182e88d763100d4788bdf421d251616040cc4d44863be"}, {file = "tqdm-4.67.0.tar.gz", hash = "sha256:fe5a6f95e6fe0b9755e9469b77b9c3cf850048224ecaa8293d7d2d31f97d869a"}, @@ -5913,14 +6308,31 @@ notebook = ["ipywidgets (>=6)"] slack = ["slack-sdk"] telegram = ["requests"] +[[package]] +name = "traitlets" +version = "5.14.3" +description = "Traitlets Python configuration system" +optional = false +python-versions = ">=3.8" +groups = ["test"] +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +files = [ + {file = "traitlets-5.14.3-py3-none-any.whl", hash = "sha256:b74e89e397b1ed28cc831db7aea759ba6640cb3de13090ca145426688ff1ac4f"}, + {file = "traitlets-5.14.3.tar.gz", hash = "sha256:9ed0579d3502c94b4b3732ac120375cda96f923114522847de4b3bb98b96b6b7"}, +] + +[package.extras] +docs = ["myst-parser", "pydata-sphinx-theme", "sphinx"] +test = ["argcomplete (>=3.0.3)", "mypy (>=1.7.0)", "pre-commit", "pytest (>=7.0,<8.2)", "pytest-mock", "pytest-mypy-testing"] + [[package]] name = "trustar" version = "0.3.34" description = "Python SDK for the TruSTAR REST API" -optional = false +optional = true python-versions = "*" -groups = ["unstable"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [] develop = false @@ -5946,10 +6358,10 @@ resolved_reference = "6954eae38e0c77eaeef26084b6c5fd033925c1c7" name = "typer" version = "0.13.1" description = "Typer, build great CLIs. Easy to code. Based on Python type hints." -optional = false +optional = true python-versions = ">=3.7" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "typer-0.13.1-py3-none-any.whl", hash = "sha256:5b59580fd925e89463a29d363e0a43245ec02765bde9fb77d39e5d0f29dd7157"}, {file = "typer-0.13.1.tar.gz", hash = "sha256:9d444cb96cc268ce6f8b94e13b4335084cef4c079998a9f4851a90229a3bd25c"}, @@ -5967,12 +6379,12 @@ version = "4.12.2" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" -groups = ["main", "unstable"] +groups = ["main", "test"] files = [ {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, ] -markers = {main = "platform_system == \"Linux\" and platform_machine == \"aarch64\"", unstable = "python_version == \"3.9\" or python_version == \"3.10\" or platform_system == \"Linux\" and python_version <= \"3.10\" and platform_machine == \"aarch64\""} +markers = {main = "extra == \"all\"", test = "python_version == \"3.9\" or python_version == \"3.11\" or python_version == \"3.10\" or platform_system == \"Linux\" and python_version < \"3.12\" and platform_machine == \"aarch64\""} [[package]] name = "tzdata" @@ -5980,12 +6392,12 @@ version = "2024.2" description = "Provider of IANA time zone data" optional = false python-versions = ">=2" -groups = ["main", "unstable"] +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and (platform_system == \"Windows\" or extra == \"all\")" files = [ {file = "tzdata-2024.2-py2.py3-none-any.whl", hash = "sha256:a48093786cdcde33cad18c2555e8532f34422074448fbc874186f0abd79565cd"}, {file = "tzdata-2024.2.tar.gz", hash = "sha256:7d85cc416e9382e69095b7bdf4afd9e3880418a2413feec7069d533d6b4e31cc"}, ] -markers = {main = "platform_system == \"Linux\" and platform_machine == \"aarch64\"", unstable = "platform_system == \"Windows\""} [[package]] name = "tzlocal" @@ -5993,7 +6405,7 @@ version = "5.2" description = "tzinfo object for the local timezone" optional = false python-versions = ">=3.8" -groups = ["main", "unstable"] +groups = ["main"] markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" files = [ {file = "tzlocal-5.2-py3-none-any.whl", hash = "sha256:49816ef2fe65ea8ac19d19aa7a1ae0551c834303d5014c6d5a62e4cbda8047b8"}, @@ -6010,35 +6422,22 @@ devenv = ["check-manifest", "pytest (>=4.3)", "pytest-cov", "pytest-mock (>=3.3) name = "unicodecsv" version = "0.14.1" description = "Python2's stdlib csv module is nice, but it doesn't support unicode. This module is a drop-in replacement which *does*." -optional = false +optional = true python-versions = "*" -groups = ["unstable"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "unicodecsv-0.14.1.tar.gz", hash = "sha256:018c08037d48649a0412063ff4eda26eaa81eff1546dbffa51fa5293276ff7fc"}, ] -[[package]] -name = "unidecode" -version = "1.3.8" -description = "ASCII transliterations of Unicode text" -optional = false -python-versions = ">=3.5" -groups = ["unstable"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" -files = [ - {file = "Unidecode-1.3.8-py3-none-any.whl", hash = "sha256:d130a61ce6696f8148a3bd8fe779c99adeb4b870584eeb9526584e9aa091fd39"}, - {file = "Unidecode-1.3.8.tar.gz", hash = "sha256:cfdb349d46ed3873ece4586b96aa75258726e2fa8ec21d6f00a591d98806c2f4"}, -] - [[package]] name = "url-normalize" version = "1.4.3" description = "URL normalization for Python" -optional = false +optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, @@ -6051,10 +6450,10 @@ six = "*" name = "urlarchiver" version = "0.2" description = "url-archiver is a simple library to fetch and archive URL on the file-system." -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "urlarchiver-0.2.tar.gz", hash = "sha256:652e0890dab58bf62a759656671dcfb9a40eb4a77aac8a8d93154f00360238b5"}, ] @@ -6069,7 +6468,7 @@ version = "1.26.20" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" -groups = ["main", "docs", "test", "unstable"] +groups = ["main", "docs", "test"] markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" files = [ {file = "urllib3-1.26.20-py2.py3-none-any.whl", hash = "sha256:0ed14ccfbf1c30a9072c7ca157e4319b70d65f623e91e7b32fadb2853431016e"}, @@ -6081,39 +6480,14 @@ brotli = ["brotli (==1.0.9) ; os_name != \"nt\" and python_version < \"3\" and p secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress ; python_version == \"2.7\"", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] -[[package]] -name = "vcrpy" -version = "7.0.0" -description = "Automatically mock your HTTP interactions to simplify and speed up testing" -optional = false -python-versions = ">=3.9" -groups = ["unstable"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" -files = [ - {file = "vcrpy-7.0.0-py2.py3-none-any.whl", hash = "sha256:55791e26c18daa363435054d8b35bd41a4ac441b6676167635d1b37a71dbe124"}, - {file = "vcrpy-7.0.0.tar.gz", hash = "sha256:176391ad0425edde1680c5b20738ea3dc7fb942520a48d2993448050986b3a50"}, -] - -[package.dependencies] -PyYAML = "*" -urllib3 = [ - {version = "<2", markers = "python_version < \"3.10\" or platform_python_implementation == \"PyPy\""}, - {version = "*", markers = "platform_python_implementation != \"PyPy\" and python_version >= \"3.10\""}, -] -wrapt = "*" -yarl = "*" - -[package.extras] -tests = ["Werkzeug (==2.0.3)", "aiohttp", "boto3", "httplib2", "httpx", "pytest", "pytest-aiohttp", "pytest-asyncio", "pytest-cov", "pytest-httpbin", "requests (>=2.22.0)", "tornado", "urllib3"] - [[package]] name = "vt-graph-api" version = "2.2.0" description = "The official Python client library for VirusTotal Graph API" -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "vt_graph_api-2.2.0-py3-none-any.whl", hash = "sha256:78da5af6a0583d0d8881c4fc3b2cb2f64075e40ac06bd47c13c414f09ff175fd"}, {file = "vt_graph_api-2.2.0.tar.gz", hash = "sha256:c44936aac6de7755c1445ed46ecc98c3c613f4b467b4738426a28101183df7ef"}, @@ -6127,10 +6501,10 @@ six = "*" name = "vt-py" version = "0.18.4" description = "The official Python client library for VirusTotal" -optional = false +optional = true python-versions = ">=3.7.0" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "vt_py-0.18.4-py3-none-any.whl", hash = "sha256:0107e2e949ac80e0010e2078e12c9cbe7ee0f1050bd2ce86a11bc155ed6769b3"}, {file = "vt_py-0.18.4.tar.gz", hash = "sha256:4ec89365d4da4d70b5be5c6bc7baa3fa12037065ef22de3eb4210d1957d31aa9"}, @@ -6146,10 +6520,10 @@ test = ["flask", "pytest", "pytest-asyncio", "pytest-httpserver"] name = "vulners" version = "2.2.3" description = "Python library and command-line utility for Vulners (https://vulners.com)" -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "vulners-2.2.3-py3-none-any.whl", hash = "sha256:fc0716989347e67655bd96ad05f7e76b444065c334e1ba351995f64e3890516d"}, {file = "vulners-2.2.3.tar.gz", hash = "sha256:ce9ff9d1410bf4bb26580c663ce001b387cb18c267fd6c5d62ab2ee5348b4353"}, @@ -6164,10 +6538,10 @@ six = ">=1.16.0,<2.0.0" name = "vysion" version = "2.0.9" description = "The official Python client library for Vysion" -optional = false +optional = true python-versions = "<4.0.0,>=3.8.0" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "vysion-2.0.9-py3-none-any.whl", hash = "sha256:f08b36a2eb67aa40f33438b6d235f0288f0ac650ea62256ded4eebf2e35a3d1c"}, {file = "vysion-2.0.9.tar.gz", hash = "sha256:503a2ca279a665ed6168bd5e1510261f4985bf967717b36ec973b52a1d98277d"}, @@ -6183,10 +6557,10 @@ softenum = "1.0.1" name = "wand" version = "0.6.13" description = "Ctypes-based simple MagickWand API binding for Python" -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "Wand-0.6.13-py2.py3-none-any.whl", hash = "sha256:e5dda0ac2204a40c29ef5c4cb310770c95d3d05c37b1379e69c94ea79d7d19c0"}, {file = "Wand-0.6.13.tar.gz", hash = "sha256:f5013484eaf7a20eb22d1821aaefe60b50cc329722372b5f8565d46d4aaafcca"}, @@ -6246,21 +6620,21 @@ version = "0.2.13" description = "Measures the displayed width of unicode strings in a terminal" optional = false python-versions = "*" -groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +groups = ["main", "test"] files = [ {file = "wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859"}, {file = "wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5"}, ] +markers = {main = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"", test = "platform_system == \"Linux\" and platform_machine == \"aarch64\""} [[package]] name = "weakrefmethod" version = "1.0.3" description = "A WeakMethod class for storing bound methods using weak references." -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "weakrefmethod-1.0.3.tar.gz", hash = "sha256:37bc1fbb5575acf82172d4eb7b6fc4412d77d5a1d70dff2c1f8a4574301cda66"}, ] @@ -6269,10 +6643,10 @@ files = [ name = "websocket-client" version = "1.8.0" description = "WebSocket client for Python with low level API options" -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "websocket_client-1.8.0-py3-none-any.whl", hash = "sha256:17b44cc997f5c498e809b22cdf2d9c7a9e71c02c8cc2b6c56e7c2d1239bfa526"}, {file = "websocket_client-1.8.0.tar.gz", hash = "sha256:3239df9f44da632f96012472805d40a23281a991027ce11d2f45a6f24ac4c3da"}, @@ -6287,10 +6661,10 @@ test = ["websockets"] name = "websockets" version = "14.1" description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" -optional = false +optional = true python-versions = ">=3.9" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "websockets-14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a0adf84bc2e7c86e8a202537b4fd50e6f7f0e4a6b6bf64d7ccb96c4cd3330b29"}, {file = "websockets-14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:90b5d9dfbb6d07a84ed3e696012610b6da074d97453bd01e0e30744b472c8179"}, @@ -6381,7 +6755,7 @@ version = "1.17.0" description = "Module for decorators, wrappers and monkey patching." optional = false python-versions = ">=3.8" -groups = ["main", "unstable"] +groups = ["main"] markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" files = [ {file = "wrapt-1.17.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2a0c23b8319848426f305f9cb0c98a6e32ee68a36264f45948ccf8e7d2b941f8"}, @@ -6455,10 +6829,10 @@ files = [ name = "wsproto" version = "1.2.0" description = "WebSockets state-machine based protocol implementation" -optional = false +optional = true python-versions = ">=3.7.0" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "wsproto-1.2.0-py3-none-any.whl", hash = "sha256:b9acddd652b585d75b20477888c56642fdade28bdfd3579aa24a4d2c037dd736"}, {file = "wsproto-1.2.0.tar.gz", hash = "sha256:ad565f26ecb92588a3e43bc3d96164de84cd9902482b130d0ddbaa9664a85065"}, @@ -6471,10 +6845,10 @@ h11 = ">=0.9.0,<1" name = "xlrd" version = "2.0.1" description = "Library for developers to extract data from Microsoft Excel (tm) .xls spreadsheet files" -optional = false +optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "xlrd-2.0.1-py2.py3-none-any.whl", hash = "sha256:6a33ee89877bd9abc1158129f6e94be74e2679636b8a205b43b85206c3f0bbdd"}, {file = "xlrd-2.0.1.tar.gz", hash = "sha256:f72f148f54442c6b056bf931dbc34f986fd0c3b0b6b5a58d013c9aef274d0c88"}, @@ -6489,10 +6863,10 @@ test = ["pytest", "pytest-cov"] name = "xlsxwriter" version = "3.2.0" description = "A Python module for creating Excel XLSX files." -optional = false +optional = true python-versions = ">=3.6" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "XlsxWriter-3.2.0-py3-none-any.whl", hash = "sha256:ecfd5405b3e0e228219bcaf24c2ca0915e012ca9464a14048021d21a995d490e"}, {file = "XlsxWriter-3.2.0.tar.gz", hash = "sha256:9977d0c661a72866a61f9f7a809e25ebbb0fb7036baa3b9fe74afcfca6b3cb8c"}, @@ -6502,10 +6876,10 @@ files = [ name = "yara-python" version = "4.5.0" description = "Python interface for YARA" -optional = false +optional = true python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "yara-python-4.5.0.tar.gz", hash = "sha256:4feecc56d2fe1d23ecb17cb2d3bc2e3859ebf7a2201d0ca3ae0756a728122b27"}, {file = "yara_python-4.5.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3feb72c2146c50e583d7e3cacbb49f280fb5cac0494cae1b48e5980ecbdc1571"}, @@ -6593,10 +6967,10 @@ files = [ name = "yarl" version = "1.18.0" description = "Yet another URL library" -optional = false +optional = true python-versions = ">=3.9" -groups = ["main", "unstable"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and extra == \"all\"" files = [ {file = "yarl-1.18.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:074fee89caab89a97e18ef5f29060ef61ba3cae6cd77673acc54bfdd3214b7b7"}, {file = "yarl-1.18.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b026cf2c32daf48d90c0c4e406815c3f8f4cfe0c6dfccb094a9add1ff6a0e41a"}, @@ -6693,12 +7067,12 @@ version = "3.21.0" description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.9" -groups = ["main", "docs", "unstable"] +groups = ["main", "docs"] files = [ {file = "zipp-3.21.0-py3-none-any.whl", hash = "sha256:ac1bbe05fd2991f160ebce24ffbac5f6d11d83dc90891255885223d42b3cd931"}, {file = "zipp-3.21.0.tar.gz", hash = "sha256:2c9958f6430a2040341a52eb608ed6dd93ef4392e02ffe219417c1b28b5dd1f4"}, ] -markers = {main = "python_version == \"3.9\" or python_version == \"3.11\" or python_version == \"3.10\" or platform_system == \"Linux\" and python_version < \"3.12\" and platform_machine == \"aarch64\"", docs = "python_version == \"3.9\" or platform_system == \"Linux\" and python_version < \"3.10\" and platform_machine == \"aarch64\"", unstable = "python_version == \"3.9\" or platform_system == \"Linux\" and python_version < \"3.10\" and platform_machine == \"aarch64\""} +markers = {main = "extra == \"all\" and python_version == \"3.9\" or extra == \"all\" and python_version == \"3.11\" or extra == \"all\" and python_version == \"3.10\" or extra == \"all\" and platform_system == \"Linux\" and python_version < \"3.12\" and platform_machine == \"aarch64\"", docs = "python_version == \"3.9\" or platform_system == \"Linux\" and python_version < \"3.10\" and platform_machine == \"aarch64\""} [package.extras] check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] @@ -6708,7 +7082,10 @@ enabler = ["pytest-enabler (>=2.2)"] test = ["big-O", "importlib-resources ; python_version < \"3.9\"", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] type = ["pytest-mypy"] +[extras] +all = ["apiosintds", "assemblyline_client", "backscatter", "blockchain", "censys", "clamd", "crowdstrike-falconpy", "dnsdb2", "domaintools_api", "geoip2", "greynoise", "jbxapi", "maclookup", "matplotlib", "mattermostdriver", "misp-lib-stix2", "misp-stix", "mwdblib", "ndjson", "np", "numpy", "oauth2", "opencv-python", "openpyxl", "pandas", "pandas_ods_reader", "pandoc", "passivetotal", "pdftotext", "pycountry", "pyeti-python3", "pyeupi", "pygeoip", "pyintel471", "pyipasnhistory", "pypdns", "pypssl", "pysafebrowsing", "pytesseract", "python-docx", "python-pptx", "pyzbar", "requests", "setuptools", "shodan", "sigmatools", "sigmf", "slack-sdk", "socialscan", "sparqlwrapper", "tau-clients", "taxii2-client", "trustar", "urlarchiver", "vt-graph-api", "vt-py", "vulners", "vysion", "wand", "xlrd", "yara-python"] + [metadata] lock-version = "2.1" python-versions = ">=3.9,<3.13" -content-hash = "2b3779a53b236f007b2c0e60b9f026e3ef09eef2bb543cea7d0f98b7bdc85891" +content-hash = "a4ed256b8ac2893a411b2c661f48b0ddae5813208c846612108c3230e50352b2" diff --git a/pyproject.toml b/pyproject.toml index a20237e39..1591d568c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "misp-modules" -version = "2.4.201" +version = "3.0.0" description = "MISP modules are autonomous modules that can be used for expansion and other services in MISP" authors = [ {name = "Alexandre Dulaunoy", email = "alexandre.dulaunoy@circl.lu"} @@ -17,12 +17,22 @@ classifiers = [ ] requires-python = ">=3.9,<3.13" dependencies = [ + ## core dependencies + "orjson", "psutil", - "pyparsing", "redis", "tornado", - "urllib3>=1.26,<2", - ## module dependencies (if a dependency fails loading with '*', pin it here) + ## minimum dependencies + "beautifulsoup4", + "jinja2", + "markdownify", + "pymisp[fileobjects, openioc, pdfexport, email]", + "setuptools", +] + +[project.optional-dependencies] +all = [ + ## pinned module dependencies "censys==2.0.9", "socialscan<2.0.0", "yara-python==4.5.0", @@ -42,7 +52,6 @@ dependencies = [ "greynoise", "jbxapi", "maclookup", - "markdownify", "matplotlib", "mattermostdriver", "misp-lib-stix2>=3.0.1.2", @@ -62,7 +71,6 @@ dependencies = [ "pygeoip", "pyintel471", "pyipasnhistory", - "pymisp[fileobjects, openioc, pdfexport, email]", "pypdns", "pypssl", "pysafebrowsing", @@ -79,6 +87,7 @@ dependencies = [ "sparqlwrapper", "tau-clients", "taxii2-client", + "trustar", "urlarchiver", "vt-graph-api", "vt-py", @@ -95,7 +104,7 @@ Documentation = "https://misp.github.io/misp-modules" Repository = "https://github.com/MISP/misp-modules" [project.scripts] -misp-modules = "misp_modules:main" +misp-modules = "misp_modules.__main__:main" [tool.poetry] packages = [{include = "misp_modules"}] @@ -108,8 +117,12 @@ build-backend = "poetry.core.masonry.api" optional = true [tool.poetry.group.test.dependencies] +black = "*" codecov = "*" flake8 = "*" +flake8-pyproject = "*" +ipdb = "*" +isort = "*" nose = "*" pytest = "*" @@ -121,16 +134,32 @@ mkdocs = "*" mkdocs-material = "*" markdown_include = "*" -[tool.poetry.group.unstable] -optional = true - -[tool.poetry.group.unstable.dependencies] -odtreader = { git = "https://github.com/cartertemm/ODTReader.git" } -google-search-api = { git = "https://github.com/abenassi/Google-Search-API" } -trustar = { git = "https://github.com/SteveClement/trustar-python.git" } -pydnstrails = { git = "https://github.com/sebdraven/pydnstrails.git" } -pyonyphe = { git = "https://github.com/sebdraven/pyonyphe.git" } - [tool.poetry.requires-plugins] poetry-plugin-bundle = ">=1.6" poetry-plugin-export = ">=1.9" + +[tool.black] +line-length = 120 +target-version = ["py39"] +preview = true +enable-unstable-feature = ["string_processing"] + +[tool.isort] +line_length = 120 +profile = "black" + +[tool.flake8] +max-line-length = 120 +exclude = [".git", "__pycache__"] +extend-ignore = [ + "E402", "E501", "E712", "E203", +] + +[tool.pytest.ini_options] +filterwarnings = [ + "ignore:.*deprecated to return a value that is not None.*:DeprecationWarning:", +] +log_cli = true +log_cli_level = "INFO" +log_cli_format = "%(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s)" +log_cli_date_format = "%Y-%m-%d %H:%M:%S" diff --git a/tests/test.py b/tests/test.py index ceac94e77..21b270319 100644 --- a/tests/test.py +++ b/tests/test.py @@ -1,29 +1,30 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -import unittest -import requests import base64 +import importlib.util +import io import json import os -import io import re +import sys +import unittest import zipfile -from hashlib import sha256 +from email.header import Header from email.mime.application import MIMEApplication -from email.mime.text import MIMEText from email.mime.multipart import MIMEMultipart -from email.header import Header +from email.mime.text import MIMEText +from hashlib import sha256 from pathlib import Path -import importlib.util -import sys + +import requests class TestModules(unittest.TestCase): def setUp(self): self.maxDiff = None - self.headers = {'Content-Type': 'application/json'} + self.headers = {"Content-Type": "application/json"} self.url = "http://127.0.0.1:6666/" def test_introspection(self): @@ -38,41 +39,46 @@ def test_introspection_module_init(self): modules_api = [module["name"] for module in response.json()] issues_found = [] root_path = Path(__file__).resolve().parent.parent - modules_path = root_path / 'misp_modules' / 'modules' + modules_path = root_path / "misp_modules" / "modules" for d in os.listdir(modules_path): - if d.startswith('__'): + if d.startswith("__"): continue mod_d_path = modules_path / d - module_files = [file[:-3] for file in os.listdir(mod_d_path) if file.endswith(".py") if file not in ['__init__.py', 'testimport.py']] + module_files = [ + file[:-3] + for file in os.listdir(mod_d_path) + if file.endswith(".py") + if file not in ["__init__.py", "testimport.py"] + ] for module in module_files: if module not in modules_api: issues_found.append(f"Missing module {module} in {d}/__init__.py.") - error_message = '\n- '.join(issues_found) + error_message = "\n- ".join(issues_found) self.assertEqual(issues_found, [], f"Found issues: \n{error_message}") finally: response.connection.close() def test_introspection_module_structure(self): moduleinfo_template = { - 'version': '1.0', - 'author': '', - 'module-type': [], - 'name': '', - 'description': '', - 'logo': '', - 'requirements': [], - 'features': '', - 'references': [], - 'input': '', - 'output': '' + "version": "1.0", + "author": "", + "module-type": [], + "name": "", + "description": "", + "logo": "", + "requirements": [], + "features": "", + "references": [], + "input": "", + "output": "", } root_path = Path(__file__).resolve().parent.parent - modules_path = root_path / 'misp_modules' / 'modules' + modules_path = root_path / "misp_modules" / "modules" issues_found = [] for d in os.listdir(modules_path): - if d.startswith('__'): + if d.startswith("__"): continue d_module = importlib.import_module(f"misp_modules.modules.{d}") @@ -89,18 +95,23 @@ def test_introspection_module_structure(self): issues_found.append(f"Error loading {module_name}: {e}") continue - sys.path.remove(str(root_path / 'misp_modules' / 'lib')) - error_message = '\n- '.join(issues_found) + sys.path.remove(str(root_path / "misp_modules" / "lib")) + error_message = "\n- ".join(issues_found) self.assertEqual(issues_found, [], f"Found issues: \n{error_message}") def test_cve(self): - with open('tests/bodycve.json', 'r') as f: + with open("tests/bodycve.json", "r") as f: response = requests.post(self.url + "query", data=f.read()) expected_response = { - 'results': [ + "results": [ { - 'types': ['text'], - 'values': 'Stack-based buffer overflow in Microsoft Office XP SP3, Office 2003 SP3, Office 2007 SP2, Office 2010, Office 2004 and 2008 for Mac, Office for Mac 2011, and Open XML File Format Converter for Mac allows remote attackers to execute arbitrary code via crafted RTF data, aka "RTF Stack Buffer Overflow Vulnerability."' + "types": ["text"], + "values": ( + "Stack-based buffer overflow in Microsoft Office XP SP3, Office 2003 SP3, Office 2007 SP2," + " Office 2010, Office 2004 and 2008 for Mac, Office for Mac 2011, and Open XML File Format" + " Converter for Mac allows remote attackers to execute arbitrary code via crafted RTF data," + ' aka "RTF Stack Buffer Overflow Vulnerability."' + ), } ] } @@ -110,24 +121,17 @@ def test_cve(self): def test_invalid_cve(self): response = requests.post(self.url + "query", data='{"module": "cve", "vulnerability": "CVE-INVALID"}') - expected_response = { - 'results': [ - { - 'types': ['text'], - 'values': 'Non existing CVE' - } - ] - } + expected_response = {"results": [{"types": ["text"], "values": "Non existing CVE"}]} self.assertDictEqual(response.json(), expected_response) print(response.json()) response.connection.close() def test_dns(self): - with open('tests/body.json', 'r') as f: + with open("tests/body.json", "r") as f: response = requests.post(self.url + "query", data=f.read()) print(response.json()) response.connection.close() - with open('tests/body_timeout.json', 'r') as f: + with open("tests/body_timeout.json", "r") as f: response = requests.post(self.url + "query", data=f.read()) print(response.json()) response.connection.close() @@ -135,30 +139,31 @@ def test_dns(self): def test_openioc(self): with open("tests/openioc.xml", "rb") as f: content = base64.b64encode(f.read()) - data = json.dumps({"module": "openiocimport", - "data": content.decode(), - }) + data = json.dumps( + { + "module": "openiocimport", + "data": content.decode(), + } + ) response = requests.post(self.url + "query", data=data).json() print(response) print("OpenIOC :: {}".format(response)) values = [x["values"][0] for x in response["results"]] - assert ("mrxcls.sys" in values) - assert ("mdmcpq3.PNF" in values) + assert "mrxcls.sys" in values + assert "mdmcpq3.PNF" in values @unittest.skip("Need Rewrite") def test_email_headers(self): query = {"module": "email_import"} - query["config"] = {"unzip_attachments": None, - "guess_zip_attachment_passwords": None, - "extract_urls": None} + query["config"] = {"unzip_attachments": None, "guess_zip_attachment_passwords": None, "extract_urls": None} message = get_base_email() text = """I am a test e-mail""" - message.attach(MIMEText(text, 'plain')) - query['data'] = decode_email(message) + message.attach(MIMEText(text, "plain")) + query["data"] = decode_email(message) data = json.dumps(query) response = requests.post(self.url + "query", data=data) - results = response.json()['results'] + results = response.json()["results"] values = [x["values"] for x in results] types = {} for i in results: @@ -166,213 +171,194 @@ def test_email_headers(self): types[i["type"]] += 1 # Check that there are the appropriate number of items # Check that all the items were correct - self.assertEqual(types['target-email'], 1) - self.assertIn('test@domain.com', values) - self.assertEqual(types['email-dst-display-name'], 4) - self.assertIn('Last One', values) - self.assertIn('Other Friend', values) - self.assertIn('Second Person', values) - self.assertIn('Testy Testerson', values) - self.assertEqual(types['email-dst'], 4) - self.assertIn('test@domain.com', values) - self.assertIn('second@domain.com', values) - self.assertIn('other@friend.net', values) - self.assertIn('last_one@finally.com', values) - self.assertEqual(types['email-src-display-name'], 2) + self.assertEqual(types["target-email"], 1) + self.assertIn("test@domain.com", values) + self.assertEqual(types["email-dst-display-name"], 4) + self.assertIn("Last One", values) + self.assertIn("Other Friend", values) + self.assertIn("Second Person", values) + self.assertIn("Testy Testerson", values) + self.assertEqual(types["email-dst"], 4) + self.assertIn("test@domain.com", values) + self.assertIn("second@domain.com", values) + self.assertIn("other@friend.net", values) + self.assertIn("last_one@finally.com", values) + self.assertEqual(types["email-src-display-name"], 2) self.assertIn("Innocent Person", values) - self.assertEqual(types['email-src'], 2) + self.assertEqual(types["email-src"], 2) self.assertIn("evil_spoofer@example.com", values) self.assertIn("IgnoreMeImInnocent@sender.com", values) - self.assertEqual(types['email-thread-index'], 1) - self.assertIn('AQHSR8Us3H3SoaY1oUy9AAwZfMF922bnA9GAgAAi9s4AAGvxAA==', values) - self.assertEqual(types['email-message-id'], 1) + self.assertEqual(types["email-thread-index"], 1) + self.assertIn("AQHSR8Us3H3SoaY1oUy9AAwZfMF922bnA9GAgAAi9s4AAGvxAA==", values) + self.assertEqual(types["email-message-id"], 1) self.assertIn("<4988EF2D.40804@example.com>", values) - self.assertEqual(types['email-subject'], 1) + self.assertEqual(types["email-subject"], 1) self.assertIn("Example Message", values) - self.assertEqual(types['email-header'], 1) - self.assertEqual(types['email-x-mailer'], 1) + self.assertEqual(types["email-header"], 1) + self.assertEqual(types["email-x-mailer"], 1) self.assertIn("mlx 5.1.7", values) - self.assertEqual(types['email-reply-to'], 1) + self.assertEqual(types["email-reply-to"], 1) self.assertIn("", values) @unittest.skip("Need Rewrite") def test_email_attachment_basic(self): query = {"module": "email_import"} - query["config"] = {"unzip_attachments": None, - "guess_zip_attachment_passwords": None, - "extract_urls": None} + query["config"] = {"unzip_attachments": None, "guess_zip_attachment_passwords": None, "extract_urls": None} message = get_base_email() text = """I am a test e-mail""" - message.attach(MIMEText(text, 'plain')) + message.attach(MIMEText(text, "plain")) with open("tests/EICAR.com", "rb") as fp: - eicar_mime = MIMEApplication(fp.read(), 'com') - eicar_mime.add_header('Content-Disposition', 'attachment', filename="EICAR.com") + eicar_mime = MIMEApplication(fp.read(), "com") + eicar_mime.add_header("Content-Disposition", "attachment", filename="EICAR.com") message.attach(eicar_mime) - query['data'] = decode_email(message) + query["data"] = decode_email(message) data = json.dumps(query) response = requests.post(self.url + "query", data=data) - values = [x["values"] for x in response.json()['results']] - self.assertIn('EICAR.com', values) - for i in response.json()['results']: - if i["type"] == 'email-attachment': + values = [x["values"] for x in response.json()["results"]] + self.assertIn("EICAR.com", values) + for i in response.json()["results"]: + if i["type"] == "email-attachment": self.assertEqual(i["values"], "EICAR.com") - if i['type'] == 'malware-sample': + if i["type"] == "malware-sample": attch_data = base64.b64decode(i["data"]) - self.assertEqual(attch_data, b'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') + self.assertEqual(attch_data, b"X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-") @unittest.skip("Need Rewrite") def test_email_attachment_unpack(self): query = {"module": "email_import"} - query["config"] = {"unzip_attachments": "true", - "guess_zip_attachment_passwords": None, - "extract_urls": None} + query["config"] = {"unzip_attachments": "true", "guess_zip_attachment_passwords": None, "extract_urls": None} message = get_base_email() text = """I am a test e-mail""" - message.attach(MIMEText(text, 'plain')) + message.attach(MIMEText(text, "plain")) with open("tests/EICAR.com.zip", "rb") as fp: - eicar_mime = MIMEApplication(fp.read(), 'zip') - eicar_mime.add_header('Content-Disposition', 'attachment', filename="EICAR.com.zip") + eicar_mime = MIMEApplication(fp.read(), "zip") + eicar_mime.add_header("Content-Disposition", "attachment", filename="EICAR.com.zip") message.attach(eicar_mime) - query['data'] = decode_email(message) + query["data"] = decode_email(message) data = json.dumps(query) response = requests.post(self.url + "query", data=data) values = [x["values"] for x in response.json()["results"]] - self.assertIn('EICAR.com', values) - self.assertIn('EICAR.com.zip', values) - for i in response.json()['results']: - if i['type'] == 'malware-sample' and i["values"] == 'EICAR.com.zip': - with zipfile.ZipFile(io.BytesIO(base64.b64decode(i["data"])), 'r') as zf: + self.assertIn("EICAR.com", values) + self.assertIn("EICAR.com.zip", values) + for i in response.json()["results"]: + if i["type"] == "malware-sample" and i["values"] == "EICAR.com.zip": + with zipfile.ZipFile(io.BytesIO(base64.b64decode(i["data"])), "r") as zf: with zf.open("EICAR.com") as ec: attch_data = ec.read() - self.assertEqual(attch_data, - b'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') - if i['type'] == 'malware-sample' and i["values"] == 'EICAR.com': + self.assertEqual(attch_data, b"X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-") + if i["type"] == "malware-sample" and i["values"] == "EICAR.com": attch_data = base64.b64decode(i["data"]) - self.assertEqual(attch_data, - b'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') + self.assertEqual(attch_data, b"X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-") @unittest.skip("Need Rewrite") def test_email_dont_unpack_compressed_doc_attachments(self): - """Ensures that compressed - """ + """Ensures that compressed""" query = {"module": "email_import"} - query["config"] = {"unzip_attachments": "true", - "guess_zip_attachment_passwords": None, - "extract_urls": None} + query["config"] = {"unzip_attachments": "true", "guess_zip_attachment_passwords": None, "extract_urls": None} message = get_base_email() text = """I am a test e-mail""" - message.attach(MIMEText(text, 'plain')) + message.attach(MIMEText(text, "plain")) with open("tests/test_files/test.docx", "rb") as fp: - eicar_mime = MIMEApplication(fp.read(), 'zip') - eicar_mime.add_header('Content-Disposition', 'attachment', filename="test.docx") + eicar_mime = MIMEApplication(fp.read(), "zip") + eicar_mime.add_header("Content-Disposition", "attachment", filename="test.docx") message.attach(eicar_mime) - query['data'] = decode_email(message) + query["data"] = decode_email(message) data = json.dumps(query) response = requests.post(self.url + "query", data=data) values = [x["values"] for x in response.json()["results"]] - self.assertIn('test.docx', values) + self.assertIn("test.docx", values) types = {} - for i in response.json()['results']: + for i in response.json()["results"]: types.setdefault(i["type"], 0) types[i["type"]] += 1 # Check that there is only one attachment in the bundle - self.assertEqual(types['malware-sample'], 1) - for i in response.json()['results']: - if i['type'] == 'malware-sample' and i["values"] == 'test.docx': + self.assertEqual(types["malware-sample"], 1) + for i in response.json()["results"]: + if i["type"] == "malware-sample" and i["values"] == "test.docx": attch_data = base64.b64decode(i["data"]) filesum = sha256() filesum.update(attch_data) - self.assertEqual(filesum.hexdigest(), - '098da5381a90d4a51e6b844c18a0fecf2e364813c2f8b317cfdc51c21f2506a5') + self.assertEqual( + filesum.hexdigest(), "098da5381a90d4a51e6b844c18a0fecf2e364813c2f8b317cfdc51c21f2506a5" + ) @unittest.skip("Need Rewrite") def test_email_attachment_unpack_with_password(self): query = {"module": "email_import"} - query["config"] = {"unzip_attachments": "true", - "guess_zip_attachment_passwords": 'true', - "extract_urls": None} + query["config"] = {"unzip_attachments": "true", "guess_zip_attachment_passwords": "true", "extract_urls": None} message = get_base_email() text = """I am a test e-mail""" - message.attach(MIMEText(text, 'plain')) + message.attach(MIMEText(text, "plain")) with open("tests/infected.zip", "rb") as fp: - eicar_mime = MIMEApplication(fp.read(), 'zip') - eicar_mime.add_header('Content-Disposition', 'attachment', filename="EICAR.com.zip") + eicar_mime = MIMEApplication(fp.read(), "zip") + eicar_mime.add_header("Content-Disposition", "attachment", filename="EICAR.com.zip") message.attach(eicar_mime) - query['data'] = decode_email(message) + query["data"] = decode_email(message) data = json.dumps(query) response = requests.post(self.url + "query", data=data) values = [x["values"] for x in response.json()["results"]] - self.assertIn('EICAR.com', values) - self.assertIn('EICAR.com.zip', values) - for i in response.json()['results']: - if i['type'] == 'malware-sample' and i["values"] == 'EICAR.com.zip': - with zipfile.ZipFile(io.BytesIO(base64.b64decode(i["data"])), 'r') as zf: + self.assertIn("EICAR.com", values) + self.assertIn("EICAR.com.zip", values) + for i in response.json()["results"]: + if i["type"] == "malware-sample" and i["values"] == "EICAR.com.zip": + with zipfile.ZipFile(io.BytesIO(base64.b64decode(i["data"])), "r") as zf: # Make sure password was set and still in place self.assertRaises(RuntimeError, zf.open, "EICAR.com") - if i['type'] == 'malware-sample' and i["values"] == 'EICAR.com': + if i["type"] == "malware-sample" and i["values"] == "EICAR.com": attch_data = base64.b64decode(i["data"]) - self.assertEqual(attch_data, - b'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') + self.assertEqual(attch_data, b"X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-") @unittest.skip("Need Rewrite") def test_email_attachment_password_in_body(self): query = {"module": "email_import"} - query["config"] = {"unzip_attachments": "true", - "guess_zip_attachment_passwords": 'true', - "extract_urls": None} + query["config"] = {"unzip_attachments": "true", "guess_zip_attachment_passwords": "true", "extract_urls": None} message = get_base_email() text = """I am a -> STRINGS <- test e-mail""" - message.attach(MIMEText(text, 'plain')) + message.attach(MIMEText(text, "plain")) with open("tests/short_password.zip", "rb") as fp: - eicar_mime = MIMEApplication(fp.read(), 'zip') - eicar_mime.add_header('Content-Disposition', 'attachment', filename="EICAR.com.zip") + eicar_mime = MIMEApplication(fp.read(), "zip") + eicar_mime.add_header("Content-Disposition", "attachment", filename="EICAR.com.zip") message.attach(eicar_mime) - query['data'] = decode_email(message) + query["data"] = decode_email(message) data = json.dumps(query) response = requests.post(self.url + "query", data=data) values = [x["values"] for x in response.json()["results"]] - self.assertIn('EICAR.com', values) - for i in response.json()['results']: - if i["values"] == 'EICAR.com': + self.assertIn("EICAR.com", values) + for i in response.json()["results"]: + if i["values"] == "EICAR.com": attch_data = base64.b64decode(i["data"]).decode() - self.assertEqual(attch_data, - 'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') + self.assertEqual(attch_data, "X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-") @unittest.skip("Need Rewrite") def test_email_attachment_password_in_body_quotes(self): query = {"module": "email_import"} - query["config"] = {"unzip_attachments": "true", - "guess_zip_attachment_passwords": 'true', - "extract_urls": None} + query["config"] = {"unzip_attachments": "true", "guess_zip_attachment_passwords": "true", "extract_urls": None} message = get_base_email() text = """I am a test e-mail the password is "a long password". That is all. """ - message.attach(MIMEText(text, 'plain')) + message.attach(MIMEText(text, "plain")) with open("tests/longer_password.zip", "rb") as fp: - eicar_mime = MIMEApplication(fp.read(), 'zip') - eicar_mime.add_header('Content-Disposition', 'attachment', filename="EICAR.com.zip") + eicar_mime = MIMEApplication(fp.read(), "zip") + eicar_mime.add_header("Content-Disposition", "attachment", filename="EICAR.com.zip") message.attach(eicar_mime) - query['data'] = decode_email(message) + query["data"] = decode_email(message) data = json.dumps(query) response = requests.post(self.url + "query", data=data) values = [x["values"] for x in response.json()["results"]] - self.assertIn('EICAR.com', values) - for i in response.json()['results']: + self.assertIn("EICAR.com", values) + for i in response.json()["results"]: # Check that it could be extracted. - if i['type'] == 'malware-sample' and i["values"] == 'EICAR.com': + if i["type"] == "malware-sample" and i["values"] == "EICAR.com": attch_data = base64.b64decode(i["data"]).decode() - self.assertEqual(attch_data, - 'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') + self.assertEqual(attch_data, "X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-") @unittest.skip("Need Rewrite") def test_email_attachment_password_in_html_body(self): query = {"module": "email_import"} - query["config"] = {"unzip_attachments": "true", - "guess_zip_attachment_passwords": 'true', - "extract_urls": None} + query["config"] = {"unzip_attachments": "true", "guess_zip_attachment_passwords": "true", "extract_urls": None} message = get_base_email() text = """I am a test e-mail the password is NOT "this string". @@ -390,30 +376,27 @@ def test_email_attachment_password_in_html_body(self): """ - message.attach(MIMEText(text, 'plain')) - message.attach(MIMEText(html, 'html')) + message.attach(MIMEText(text, "plain")) + message.attach(MIMEText(html, "html")) with open("tests/longer_password.zip", "rb") as fp: - eicar_mime = MIMEApplication(fp.read(), 'zip') - eicar_mime.add_header('Content-Disposition', 'attachment', filename="EICAR.com.zip") + eicar_mime = MIMEApplication(fp.read(), "zip") + eicar_mime.add_header("Content-Disposition", "attachment", filename="EICAR.com.zip") message.attach(eicar_mime) - query['data'] = decode_email(message) + query["data"] = decode_email(message) data = json.dumps(query) response = requests.post(self.url + "query", data=data) values = [x["values"] for x in response.json()["results"]] - self.assertIn('EICAR.com', values) - for i in response.json()['results']: + self.assertIn("EICAR.com", values) + for i in response.json()["results"]: # Check that it could be extracted. - if i["values"] == 'EICAR.com': + if i["values"] == "EICAR.com": attch_data = base64.b64decode(i["data"]).decode() - self.assertEqual(attch_data, - 'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') + self.assertEqual(attch_data, "X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-") @unittest.skip("Need Rewrite") def test_email_body_encoding(self): query = {"module": "email_import"} - query["config"] = {"unzip_attachments": None, - "guess_zip_attachment_passwords": None, - "extract_urls": None} + query["config"] = {"unzip_attachments": None, "guess_zip_attachment_passwords": None, "extract_urls": None} filenames = os.listdir("tests/test_files/encodings") for fn in filenames: message = get_base_email() @@ -421,40 +404,38 @@ def test_email_body_encoding(self): with open("tests/test_files/encodings/{0}".format(fn), "r", encoding=encoding[0]) as fp: # Encoding is used as the name of the file text = fp.read() - message.attach(MIMEText(text, 'html', encoding[0])) - query['data'] = decode_email(message) + message.attach(MIMEText(text, "html", encoding[0])) + query["data"] = decode_email(message) data = json.dumps(query) response = requests.post(self.url + "query", data=data).json() - self.assertNotIn('error', response, response.get('error', "")) - self.assertIn('results', response, "No server results found.") + self.assertNotIn("error", response, response.get("error", "")) + self.assertIn("results", response, "No server results found.") @unittest.skip("Need Rewrite") def test_email_header_proper_encoding(self): query = {"module": "email_import"} - query["config"] = {"unzip_attachments": None, - "guess_zip_attachment_passwords": None, - "extract_urls": None} + query["config"] = {"unzip_attachments": None, "guess_zip_attachment_passwords": None, "extract_urls": None} filenames = os.listdir("tests/test_files/encodings") - for encoding in ['utf-8', 'utf-16', 'utf-32']: + for encoding in ["utf-8", "utf-16", "utf-32"]: message = get_base_email() text = """I am a test e-mail the password is NOT "this string". That is all. """ - message.attach(MIMEText(text, 'plain')) + message.attach(MIMEText(text, "plain")) for hdr, hdr_val in message.items(): msg = message encoded_header = hdr_val.encode(encoding) msg.replace_header(hdr, Header(encoded_header, encoding)) - query['data'] = decode_email(msg) + query["data"] = decode_email(msg) data = json.dumps(query) response = requests.post(self.url + "query", data=data) - results = response.json()['results'] + results = response.json()["results"] values = [] for x in results: # Remove BOM from UTF-16 strings - if re.search('\ufeff', x["values"]): - values.append(re.sub('\ufeff', "", x["values"])) + if re.search("\ufeff", x["values"]): + values.append(re.sub("\ufeff", "", x["values"])) else: values.append(x["values"]) types = {} @@ -462,33 +443,33 @@ def test_email_header_proper_encoding(self): types.setdefault(i["type"], 0) types[i["type"]] += 1 # Check that all the items were correct - self.assertEqual(types['target-email'], 1) - self.assertIn('test@domain.com', values) - self.assertEqual(types['email-dst-display-name'], 4) - self.assertIn('Last One', values) - self.assertIn('Other Friend', values) - self.assertIn('Second Person', values) - self.assertIn('Testy Testerson', values) - self.assertEqual(types['email-dst'], 4) - self.assertIn('test@domain.com', values) - self.assertIn('second@domain.com', values) - self.assertIn('other@friend.net', values) - self.assertIn('last_one@finally.com', values) - self.assertEqual(types['email-src-display-name'], 2) + self.assertEqual(types["target-email"], 1) + self.assertIn("test@domain.com", values) + self.assertEqual(types["email-dst-display-name"], 4) + self.assertIn("Last One", values) + self.assertIn("Other Friend", values) + self.assertIn("Second Person", values) + self.assertIn("Testy Testerson", values) + self.assertEqual(types["email-dst"], 4) + self.assertIn("test@domain.com", values) + self.assertIn("second@domain.com", values) + self.assertIn("other@friend.net", values) + self.assertIn("last_one@finally.com", values) + self.assertEqual(types["email-src-display-name"], 2) self.assertIn("Innocent Person", values) - self.assertEqual(types['email-src'], 2) + self.assertEqual(types["email-src"], 2) self.assertIn("evil_spoofer@example.com", values) self.assertIn("IgnoreMeImInnocent@sender.com", values) - self.assertEqual(types['email-thread-index'], 1) - self.assertIn('AQHSR8Us3H3SoaY1oUy9AAwZfMF922bnA9GAgAAi9s4AAGvxAA==', values) - self.assertEqual(types['email-message-id'], 1) + self.assertEqual(types["email-thread-index"], 1) + self.assertIn("AQHSR8Us3H3SoaY1oUy9AAwZfMF922bnA9GAgAAi9s4AAGvxAA==", values) + self.assertEqual(types["email-message-id"], 1) self.assertIn("<4988EF2D.40804@example.com>", values) - self.assertEqual(types['email-subject'], 1) + self.assertEqual(types["email-subject"], 1) self.assertIn("Example Message", values) - self.assertEqual(types['email-header'], 1) - self.assertEqual(types['email-x-mailer'], 1) + self.assertEqual(types["email-header"], 1) + self.assertEqual(types["email-x-mailer"], 1) self.assertIn("mlx 5.1.7", values) - self.assertEqual(types['email-reply-to'], 1) + self.assertEqual(types["email-reply-to"], 1) self.assertIn("", values) self.assertIn("", values) @@ -496,33 +477,31 @@ def test_email_header_proper_encoding(self): @unittest.skip("Need Rewrite") def test_email_header_malformed_encoding(self): query = {"module": "email_import"} - query["config"] = {"unzip_attachments": None, - "guess_zip_attachment_passwords": None, - "extract_urls": None} + query["config"] = {"unzip_attachments": None, "guess_zip_attachment_passwords": None, "extract_urls": None} filenames = os.listdir("tests/test_files/encodings") - for encoding in ['utf-8', 'utf-16', 'utf-32']: + for encoding in ["utf-8", "utf-16", "utf-32"]: message = get_base_email() text = """I am a test e-mail the password is NOT "this string". That is all. """ - message.attach(MIMEText(text, 'plain')) + message.attach(MIMEText(text, "plain")) for hdr, hdr_val in message.items(): msg = message encoded_header = hdr_val.encode(encoding) pat = re.compile(hdr_val.encode()) message_bytes = pat.sub(encoded_header, msg.as_bytes()) message64 = base64.b64encode(message_bytes).decode() - query['data'] = message64 + query["data"] = message64 data = json.dumps(query) response = requests.post(self.url + "query", data=data) - results = response.json()['results'] + results = response.json()["results"] values = [] for x in results: # Remove BOM from UTF-16 strings - if re.search('\ufeff', x["values"]): - values.append(re.sub('\ufeff', "", x["values"])) + if re.search("\ufeff", x["values"]): + values.append(re.sub("\ufeff", "", x["values"])) else: values.append(x["values"]) types = {} @@ -530,33 +509,33 @@ def test_email_header_malformed_encoding(self): types.setdefault(i["type"], 0) types[i["type"]] += 1 # Check that all the items were correct - self.assertEqual(types['target-email'], 1) - self.assertIn('test@domain.com', values) - self.assertEqual(types['email-dst-display-name'], 4) - self.assertIn('Last One', values) - self.assertIn('Other Friend', values) - self.assertIn('Second Person', values) - self.assertIn('Testy Testerson', values) - self.assertEqual(types['email-dst'], 4) - self.assertIn('test@domain.com', values) - self.assertIn('second@domain.com', values) - self.assertIn('other@friend.net', values) - self.assertIn('last_one@finally.com', values) - self.assertEqual(types['email-src-display-name'], 2) + self.assertEqual(types["target-email"], 1) + self.assertIn("test@domain.com", values) + self.assertEqual(types["email-dst-display-name"], 4) + self.assertIn("Last One", values) + self.assertIn("Other Friend", values) + self.assertIn("Second Person", values) + self.assertIn("Testy Testerson", values) + self.assertEqual(types["email-dst"], 4) + self.assertIn("test@domain.com", values) + self.assertIn("second@domain.com", values) + self.assertIn("other@friend.net", values) + self.assertIn("last_one@finally.com", values) + self.assertEqual(types["email-src-display-name"], 2) self.assertIn("Innocent Person", values) - self.assertEqual(types['email-src'], 2) + self.assertEqual(types["email-src"], 2) self.assertIn("evil_spoofer@example.com", values) self.assertIn("IgnoreMeImInnocent@sender.com", values) - self.assertEqual(types['email-thread-index'], 1) - self.assertIn('AQHSR8Us3H3SoaY1oUy9AAwZfMF922bnA9GAgAAi9s4AAGvxAA==', values) - self.assertEqual(types['email-message-id'], 1) + self.assertEqual(types["email-thread-index"], 1) + self.assertIn("AQHSR8Us3H3SoaY1oUy9AAwZfMF922bnA9GAgAAi9s4AAGvxAA==", values) + self.assertEqual(types["email-message-id"], 1) self.assertIn("<4988EF2D.40804@example.com>", values) - self.assertEqual(types['email-subject'], 1) + self.assertEqual(types["email-subject"], 1) self.assertIn("Example Message", values) - self.assertEqual(types['email-header'], 1) - self.assertEqual(types['email-x-mailer'], 1) + self.assertEqual(types["email-header"], 1) + self.assertEqual(types["email-x-mailer"], 1) self.assertIn("mlx 5.1.7", values) - self.assertEqual(types['email-reply-to'], 1) + self.assertEqual(types["email-reply-to"], 1) self.assertIn("", values) self.assertIn("", values) @@ -564,9 +543,7 @@ def test_email_header_malformed_encoding(self): @unittest.skip("Need Rewrite") def test_email_header_CJK_encoding(self): query = {"module": "email_import"} - query["config"] = {"unzip_attachments": None, - "guess_zip_attachment_passwords": None, - "extract_urls": None} + query["config"] = {"unzip_attachments": None, "guess_zip_attachment_passwords": None, "extract_urls": None} # filenames = os.listdir("tests/test_files/encodings") # for encoding in ['utf-8', 'utf-16', 'utf-32']: message = get_base_email() @@ -574,27 +551,25 @@ def test_email_header_CJK_encoding(self): the password is NOT "this string". That is all. """ - message.attach(MIMEText(text, 'plain')) + message.attach(MIMEText(text, "plain")) japanese_charset = "ビット及び8ビットの2バイト情報交換用符号化拡張漢字集合" - jisx213 = Header(japanese_charset, 'euc_jisx0213') + jisx213 = Header(japanese_charset, "euc_jisx0213") message.replace_header("Subject", jisx213) - query['data'] = decode_email(message) + query["data"] = decode_email(message) data = json.dumps(query) response = requests.post(self.url + "query", data=data) # Parse Response - RFC_format = '=?euc_jisx0213?b?pdOlw6XItdqk0zil06XDpcikzjKl0KWkpci+8MrzuPK0uc3RyeS55rK9s8jEpbTBu/q9uLnn?=' - for i in response.json()['results']: - if i['type'] == 'email-subject': + RFC_format = "=?euc_jisx0213?b?pdOlw6XItdqk0zil06XDpcikzjKl0KWkpci+8MrzuPK0uc3RyeS55rK9s8jEpbTBu/q9uLnn?=" + for i in response.json()["results"]: + if i["type"] == "email-subject": RFC_encoding_error = "The subject was not decoded from RFC2047 format." - self.assertNotEqual(RFC_format, i['values'], RFC_encoding_error) - self.assertEqual(japanese_charset, i['values'], "Subject not properly decoded") + self.assertNotEqual(RFC_format, i["values"], RFC_encoding_error) + self.assertEqual(japanese_charset, i["values"], "Subject not properly decoded") @unittest.skip("Need Rewrite") def test_email_malformed_header_CJK_encoding(self): query = {"module": "email_import"} - query["config"] = {"unzip_attachments": None, - "guess_zip_attachment_passwords": None, - "extract_urls": None} + query["config"] = {"unzip_attachments": None, "guess_zip_attachment_passwords": None, "extract_urls": None} # filenames = os.listdir("tests/test_files/encodings") # for encoding in ['utf-8', 'utf-16', 'utf-32']: message = get_base_email() @@ -602,30 +577,28 @@ def test_email_malformed_header_CJK_encoding(self): the password is NOT "this string". That is all. """ - message.attach(MIMEText(text, 'plain')) + message.attach(MIMEText(text, "plain")) japanese_charset = "ビット及び8ビットの2バイト情報交換用符号化拡張漢字集合" japanese_bytes = japanese_charset.encode() - message.replace_header('Subject', "{{REPLACE}}") - pat = re.compile(b'{{REPLACE}}') + message.replace_header("Subject", "{{REPLACE}}") + pat = re.compile(b"{{REPLACE}}") message_bytes = pat.sub(japanese_bytes, message.as_bytes()) message64 = base64.b64encode(message_bytes).decode() - query['data'] = message64 + query["data"] = message64 data = json.dumps(query) response = requests.post(self.url + "query", data=data) # Parse Response - RFC_format = '=?euc_jisx0213?b?pdOlw6XItdqk0zil06XDpcikzjKl0KWkpci+8MrzuPK0uc3RyeS55rK9s8jEpbTBu/q9uLnn?=' - for i in response.json()['results']: - if i['type'] == 'email-subject': + RFC_format = "=?euc_jisx0213?b?pdOlw6XItdqk0zil06XDpcikzjKl0KWkpci+8MrzuPK0uc3RyeS55rK9s8jEpbTBu/q9uLnn?=" + for i in response.json()["results"]: + if i["type"] == "email-subject": RFC_encoding_error = "The subject was not decoded from RFC2047 format." - self.assertNotEqual(RFC_format, i['values'], RFC_encoding_error) - self.assertEqual(japanese_charset, i['values'], "Subject not properly decoded") + self.assertNotEqual(RFC_format, i["values"], RFC_encoding_error) + self.assertEqual(japanese_charset, i["values"], "Subject not properly decoded") @unittest.skip("Need Rewrite") def test_email_malformed_header_emoji_encoding(self): query = {"module": "email_import"} - query["config"] = {"unzip_attachments": None, - "guess_zip_attachment_passwords": None, - "extract_urls": None} + query["config"] = {"unzip_attachments": None, "guess_zip_attachment_passwords": None, "extract_urls": None} # filenames = os.listdir("tests/test_files/encodings") # for encoding in ['utf-8', 'utf-16', 'utf-32']: message = get_base_email() @@ -633,57 +606,51 @@ def test_email_malformed_header_emoji_encoding(self): the password is NOT "this string". That is all. """ - message.attach(MIMEText(text, 'plain')) + message.attach(MIMEText(text, "plain")) emoji_string = "Emoji Test 👍 checking this" emoji_bytes = emoji_string.encode() - message.replace_header('Subject', "{{EMOJI}}") - pat = re.compile(b'{{EMOJI}}') + message.replace_header("Subject", "{{EMOJI}}") + pat = re.compile(b"{{EMOJI}}") message_bytes = pat.sub(emoji_bytes, message.as_bytes()) message64 = base64.b64encode(message_bytes).decode() - query['data'] = message64 + query["data"] = message64 data = json.dumps(query) response = requests.post(self.url + "query", data=data) # Parse Response RFC_format = "=?unknown-8bit?q?Emoji_Test_=F0=9F=91=8D_checking_this?=" - for i in response.json()['results']: - if i['type'] == 'email-subject': + for i in response.json()["results"]: + if i["type"] == "email-subject": RFC_encoding_error = "The subject was not decoded from RFC2047 format." - self.assertNotEqual(RFC_format, i['values'], RFC_encoding_error) - self.assertEqual(emoji_string, i['values'], "Subject not properly decoded") + self.assertNotEqual(RFC_format, i["values"], RFC_encoding_error) + self.assertEqual(emoji_string, i["values"], "Subject not properly decoded") @unittest.skip("Need Rewrite") def test_email_attachment_emoji_filename(self): query = {"module": "email_import"} - query["config"] = {"unzip_attachments": None, - "guess_zip_attachment_passwords": None, - "extract_urls": None} + query["config"] = {"unzip_attachments": None, "guess_zip_attachment_passwords": None, "extract_urls": None} message = get_base_email() text = """I am a test e-mail""" - message.attach(MIMEText(text, 'plain')) + message.attach(MIMEText(text, "plain")) with open("tests/EICAR.com", "rb") as fp: - eicar_mime = MIMEApplication(fp.read(), 'com') - eicar_mime.add_header('Content-Disposition', - 'attachment', - filename="Emoji Test 👍 checking this") + eicar_mime = MIMEApplication(fp.read(), "com") + eicar_mime.add_header("Content-Disposition", "attachment", filename="Emoji Test 👍 checking this") message.attach(eicar_mime) - query['data'] = decode_email(message) + query["data"] = decode_email(message) data = json.dumps(query) response = requests.post(self.url + "query", data=data) - values = [x["values"] for x in response.json()['results']] + values = [x["values"] for x in response.json()["results"]] self.assertIn("Emoji Test 👍 checking this", values) - for i in response.json()['results']: - if i["type"] == 'email-attachment': + for i in response.json()["results"]: + if i["type"] == "email-attachment": self.assertEqual(i["values"], "Emoji Test 👍 checking this") - if i['type'] == 'malware-sample': + if i["type"] == "malware-sample": attch_data = base64.b64decode(i["data"]) - self.assertEqual(attch_data, b'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') + self.assertEqual(attch_data, b"X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-") @unittest.skip("Need Rewrite") def test_email_attachment_password_in_subject(self): query = {"module": "email_import"} - query["config"] = {"unzip_attachments": "true", - "guess_zip_attachment_passwords": 'true', - "extract_urls": None} + query["config"] = {"unzip_attachments": "true", "guess_zip_attachment_passwords": "true", "extract_urls": None} message = get_base_email() message.replace_header("Subject", 'I contain the -> "a long password" <- that is the password') text = """I am a test e-mail @@ -691,30 +658,27 @@ def test_email_attachment_password_in_subject(self): That is all. """ - message.attach(MIMEText(text, 'plain')) + message.attach(MIMEText(text, "plain")) with open("tests/longer_password.zip", "rb") as fp: - eicar_mime = MIMEApplication(fp.read(), 'zip') - eicar_mime.add_header('Content-Disposition', 'attachment', filename="EICAR.com.zip") + eicar_mime = MIMEApplication(fp.read(), "zip") + eicar_mime.add_header("Content-Disposition", "attachment", filename="EICAR.com.zip") message.attach(eicar_mime) - query['data'] = decode_email(message) + query["data"] = decode_email(message) data = json.dumps(query) response = requests.post(self.url + "query", data=data) values = [x["values"] for x in response.json()["results"]] - self.assertIn('EICAR.com', values) + self.assertIn("EICAR.com", values) self.assertIn('I contain the -> "a long password" <- that is the password', values) - for i in response.json()['results']: + for i in response.json()["results"]: # Check that it could be extracted. - if i["values"] == 'EICAR.com': + if i["values"] == "EICAR.com": attch_data = base64.b64decode(i["data"]).decode() - self.assertEqual(attch_data, - 'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') + self.assertEqual(attch_data, "X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-") @unittest.skip("Need Rewrite") def test_email_extract_html_body_urls(self): query = {"module": "email_import"} - query["config"] = {"unzip_attachments": None, - "guess_zip_attachment_passwords": None, - "extract_urls": "true"} + query["config"] = {"unzip_attachments": None, "guess_zip_attachment_passwords": None, "extract_urls": "true"} message = get_base_email() text = """I am a test e-mail @@ -734,9 +698,9 @@ def test_email_extract_html_body_urls(self): """ - message.attach(MIMEText(text, 'plain')) - message.attach(MIMEText(html, 'html')) - query['data'] = decode_email(message) + message.attach(MIMEText(text, "plain")) + message.attach(MIMEText(html, "html")) + query["data"] = decode_email(message) data = json.dumps(query) response = requests.post(self.url + "query", data=data) # print(response.json()) @@ -760,32 +724,45 @@ def decode_email(message): def get_base_email(): - headers = {"Received": "via dmail-2008.19 for +INBOX; Tue, 3 Feb 2009 19:29:12 -0600 (CST)", - "Received": "from abc.luxsci.com ([10.10.10.10]) by xyz.luxsci.com (8.13.7/8.13.7) with ESMTP id n141TCa7022588 for ; Tue, 3 Feb 2009 19:29:12 -0600", - "Received": "from [192.168.0.3] (verizon.net [44.44.44.44]) (user=test@sender.com mech=PLAIN bits=2) by abc.luxsci.com (8.13.7/8.13.7) with ESMTP id n141SAfo021855 (version=TLSv1/SSLv3 cipher=DHE-RSA-AES256-SHA bits=256 verify=NOT) for ; Tue, 3 Feb 2009 19:28:10 -0600", - "X-Received": "by 192.168.0.45 with SMTP id q4mr156123401yw1g.911.1912342394963; Tue, 3 Feb 2009 19:32:15 -0600 (PST)", - "Message-ID": "<4988EF2D.40804@example.com>", - "Date": "Tue, 03 Feb 2009 20:28:13 -0500", - "From": '"Innocent Person" ', - "User-Agent": 'Thunderbird 2.0.0.19 (Windows/20081209)', - "Sender": '"Malicious MailAgent" ', - "References": "", - "In-Reply-To": "", - "Accept-Language": 'en-US', - "X-Mailer": 'mlx 5.1.7', - "Return-Path": "evil_spoofer@example.com", - "Thread-Topic": 'This is a thread.', - "Thread-Index": 'AQHSR8Us3H3SoaY1oUy9AAwZfMF922bnA9GAgAAi9s4AAGvxAA==', - "Content-Language": 'en-US', - "To": '"Testy Testerson" ', - "Cc": '"Second Person" , "Other Friend" , "Last One" ', - "Subject": 'Example Message', - "MIME-Version": '1.0'} + headers = { + "Received": "via dmail-2008.19 for +INBOX; Tue, 3 Feb 2009 19:29:12 -0600 (CST)", + "Received": ( + "from abc.luxsci.com ([10.10.10.10]) by xyz.luxsci.com (8.13.7/8.13.7) with ESMTP id n141TCa7022588 for" + " ; Tue, 3 Feb 2009 19:29:12 -0600" + ), + "Received": ( + "from [192.168.0.3] (verizon.net [44.44.44.44]) (user=test@sender.com mech=PLAIN bits=2) by abc.luxsci.com" + " (8.13.7/8.13.7) with ESMTP id n141SAfo021855 (version=TLSv1/SSLv3 cipher=DHE-RSA-AES256-SHA bits=256" + " verify=NOT) for ; Tue, 3 Feb 2009 19:28:10 -0600" + ), + "X-Received": ( + "by 192.168.0.45 with SMTP id q4mr156123401yw1g.911.1912342394963; Tue, 3 Feb 2009 19:32:15 -0600 (PST)" + ), + "Message-ID": "<4988EF2D.40804@example.com>", + "Date": "Tue, 03 Feb 2009 20:28:13 -0500", + "From": '"Innocent Person" ', + "User-Agent": "Thunderbird 2.0.0.19 (Windows/20081209)", + "Sender": '"Malicious MailAgent" ', + "References": "", + "In-Reply-To": "", + "Accept-Language": "en-US", + "X-Mailer": "mlx 5.1.7", + "Return-Path": "evil_spoofer@example.com", + "Thread-Topic": "This is a thread.", + "Thread-Index": "AQHSR8Us3H3SoaY1oUy9AAwZfMF922bnA9GAgAAi9s4AAGvxAA==", + "Content-Language": "en-US", + "To": '"Testy Testerson" ', + "Cc": ( + '"Second Person" , "Other Friend" , "Last One" ' + ), + "Subject": "Example Message", + "MIME-Version": "1.0", + } msg = MIMEMultipart() for key, val in headers.items(): msg.add_header(key, val) return msg -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/test_actions.py b/tests/test_actions.py index 56f16cb3b..3ff06b3c7 100644 --- a/tests/test_actions.py +++ b/tests/test_actions.py @@ -1,12 +1,14 @@ import os import unittest + import requests class TestActions(unittest.TestCase): """Unittest module for action modules""" + def setUp(self): - self.headers = {'Content-Type': 'application/json'} + self.headers = {"Content-Type": "application/json"} self.url = "http://127.0.0.1:6666/" def test_introspection(self): @@ -15,8 +17,12 @@ def test_introspection(self): response = requests.get(self.url + "modules") modules = [module["name"] for module in response.json()] # list modules in the export_mod folder - export_mod_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'misp_modules', 'modules', "action_mod") - module_files = [file[:-3] for file in os.listdir(export_mod_path) if file.endswith(".py") if file not in ['__init__.py']] + export_mod_path = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "..", "misp_modules", "modules", "action_mod" + ) + module_files = [ + file[:-3] for file in os.listdir(export_mod_path) if file.endswith(".py") if file not in ["__init__.py"] + ] missing = [] for module in module_files: if module not in modules: diff --git a/tests/test_expansions.py b/tests/test_expansions.py index 2ff165ee5..8a81a26d9 100644 --- a/tests/test_expansions.py +++ b/tests/test_expansions.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -import unittest -import requests -from urllib.parse import urljoin -from base64 import b64encode import json import os +import unittest +from base64 import b64encode +from urllib.parse import urljoin + +import requests LiveCI = True @@ -15,12 +16,22 @@ class TestExpansions(unittest.TestCase): def setUp(self): self.maxDiff = None - self.headers = {'Content-Type': 'application/json'} + self.headers = {"Content-Type": "application/json"} self.url = "http://127.0.0.1:6666/" self.dirname = os.path.dirname(os.path.realpath(__file__)) - self.sigma_rule = "title: Antivirus Web Shell Detection\r\ndescription: Detects a highly relevant Antivirus alert that reports a web shell\r\ndate: 2018/09/09\r\nmodified: 2019/10/04\r\nauthor: Florian Roth\r\nreferences:\r\n - https://www.nextron-systems.com/2018/09/08/antivirus-event-analysis-cheat-sheet-v1-4/\r\ntags:\r\n - attack.persistence\r\n - attack.t1100\r\nlogsource:\r\n product: antivirus\r\ndetection:\r\n selection:\r\n Signature: \r\n - \"PHP/Backdoor*\"\r\n - \"JSP/Backdoor*\"\r\n - \"ASP/Backdoor*\"\r\n - \"Backdoor.PHP*\"\r\n - \"Backdoor.JSP*\"\r\n - \"Backdoor.ASP*\"\r\n - \"*Webshell*\"\r\n condition: selection\r\nfields:\r\n - FileName\r\n - User\r\nfalsepositives:\r\n - Unlikely\r\nlevel: critical" + self.sigma_rule = ( + "title: Antivirus Web Shell Detection\r\ndescription: Detects a highly relevant Antivirus alert that" + " reports a web shell\r\ndate: 2018/09/09\r\nmodified: 2019/10/04\r\nauthor: Florian" + " Roth\r\nreferences:\r\n -" + " https://www.nextron-systems.com/2018/09/08/antivirus-event-analysis-cheat-sheet-v1-4/\r\ntags:\r\n -" + " attack.persistence\r\n - attack.t1100\r\nlogsource:\r\n product: antivirus\r\ndetection:\r\n " + ' selection:\r\n Signature: \r\n - "PHP/Backdoor*"\r\n - "JSP/Backdoor*"\r\n ' + ' - "ASP/Backdoor*"\r\n - "Backdoor.PHP*"\r\n - "Backdoor.JSP*"\r\n ' + ' - "Backdoor.ASP*"\r\n - "*Webshell*"\r\n condition: selection\r\nfields:\r\n -' + " FileName\r\n - User\r\nfalsepositives:\r\n - Unlikely\r\nlevel: critical" + ) try: - with open(f'{self.dirname}/expansion_configs.json', 'rb') as f: + with open(f"{self.dirname}/expansion_configs.json", "rb") as f: self.configs = json.loads(f.read().decode()) except FileNotFoundError: self.configs = {} @@ -35,8 +46,8 @@ def get_attribute_types(response): print(json.dumps(data, indent=2)) return data types = [] - for attribute in data['results']['Attribute']: - types.append(attribute['type']) + for attribute in data["results"]["Attribute"]: + types.append(attribute["type"]) return types @staticmethod @@ -45,7 +56,7 @@ def get_data(response): if not isinstance(data, dict): print(json.dumps(data, indent=2)) return data - return data['results'][0]['data'] + return data["results"][0]["data"] @staticmethod def get_errors(response): @@ -53,7 +64,7 @@ def get_errors(response): if not isinstance(data, dict): print(json.dumps(data, indent=2)) return data - return data['error'] + return data["error"] @staticmethod def get_object_types(response): @@ -62,8 +73,8 @@ def get_object_types(response): print(json.dumps(data, indent=2)) return data names = [] - for obj in data['results']['Object']: - names.append(obj['name']) + for obj in data["results"]["Object"]: + names.append(obj["name"]) return names @staticmethod @@ -72,7 +83,7 @@ def get_first_object_type(response): if not isinstance(data, dict): print(json.dumps(data, indent=2)) return data - return data['results']['Object'][0]['name'] + return data["results"]["Object"][0]["name"] @staticmethod def get_values(response): @@ -80,13 +91,13 @@ def get_values(response): if not isinstance(data, dict): print(json.dumps(data, indent=2)) return data - if 'results' not in data: + if "results" not in data: return data - for result in data['results']: - values = result['values'] + for result in data["results"]: + values = result["values"] if values: return values[0] if isinstance(values, list) else values - return data['results'][0]['values'] + return data["results"][0]["values"] def test_introspection(self): """checks if all expansion modules are offered through the misp-modules service""" @@ -94,8 +105,12 @@ def test_introspection(self): response = requests.get(self.url + "modules") modules = [module["name"] for module in response.json()] # list modules in the export_mod folder - export_mod_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'misp_modules', 'modules', "expansion") - module_files = [file[:-3] for file in os.listdir(export_mod_path) if file.endswith(".py") if file not in ['__init__.py']] + export_mod_path = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "..", "misp_modules", "modules", "expansion" + ) + module_files = [ + file[:-3] for file in os.listdir(export_mod_path) if file.endswith(".py") if file not in ["__init__.py"] + ] missing = [] for module in module_files: if module not in modules: @@ -107,31 +122,31 @@ def test_introspection(self): def test_apiosintds(self): self.skipTest("apiosintds is probably broken") - query = {'module': 'apiosintds', 'ip-dst': '10.10.10.10'} + query = {"module": "apiosintds", "ip-dst": "10.10.10.10"} response = self.misp_modules_post(query) try: - self.assertTrue(self.get_values(response).startswith('IoC 10.10.10.10')) + self.assertTrue(self.get_values(response).startswith("IoC 10.10.10.10")) except AssertionError: - self.assertTrue(self.get_values(response).startswith('10.10.10.10 IS NOT listed by OSINT.digitalside.it.')) + self.assertTrue(self.get_values(response).startswith("10.10.10.10 IS NOT listed by OSINT.digitalside.it.")) def test_apivoid(self): module_name = "apivoid" - query = {"module": module_name, - "attribute": {"type": "domain", - "value": "circl.lu", - "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}, - "config": {}} + query = { + "module": module_name, + "attribute": {"type": "domain", "value": "circl.lu", "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}, + "config": {}, + } if module_name in self.configs: - query['config'] = self.configs[module_name] + query["config"] = self.configs[module_name] response = self.misp_modules_post(query) try: - self.assertEqual(self.get_first_object_type(response), 'dns-record') + self.assertEqual(self.get_first_object_type(response), "dns-record") except Exception: - self.assertTrue(self.get_errors(response).startswith('You do not have enough APIVoid credits')) + self.assertTrue(self.get_errors(response).startswith("You do not have enough APIVoid credits")) else: response = self.misp_modules_post(query) - self.assertEqual(self.get_errors(response), 'An API key for APIVoid is required.') + self.assertEqual(self.get_errors(response), "An API key for APIVoid is required.") def test_btc_steroids(self): if LiveCI: @@ -140,60 +155,68 @@ def test_btc_steroids(self): query = {"module": "btc_steroids", "btc": "1ES14c7qLb5CYhLMUekctxLgc1FV2Ti9DA"} response = self.misp_modules_post(query) try: - self.assertTrue(self.get_values(response).startswith('\n\nAddress:\t1ES14c7qLb5CYhLMUekctxLgc1FV2Ti9DA\nBalance:\t0.0002126800 BTC (+0.0007482500 BTC / -0.0005355700 BTC)')) + self.assertTrue( + self.get_values(response).startswith( + "\n\nAddress:\t1ES14c7qLb5CYhLMUekctxLgc1FV2Ti9DA\nBalance:\t0.0002126800 BTC (+0.0007482500 BTC /" + " -0.0005355700 BTC)" + ) + ) except Exception: - self.assertTrue(self.get_values(response).startswith('Not a valid BTC address')) + self.assertTrue(self.get_values(response).startswith("Not a valid BTC address")) def test_btc_scam_check(self): query = {"module": "btc_scam_check", "btc": "1ES14c7qLb5CYhLMUekctxLgc1FV2Ti9DA"} response = self.misp_modules_post(query) - self.assertEqual(self.get_values(response), '1es14c7qlb5cyhlmuekctxlgc1fv2ti9da fraudolent bitcoin address') + self.assertEqual(self.get_values(response), "1es14c7qlb5cyhlmuekctxlgc1fv2ti9da fraudolent bitcoin address") def test_circl_passivedns(self): module_name = "circl_passivedns" - query = {"module": module_name, - "attribute": {"type": "domain", - "value": "circl.lu", - "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}, - "config": {}} + query = { + "module": module_name, + "attribute": {"type": "domain", "value": "circl.lu", "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}, + "config": {}, + } if module_name in self.configs: - query['config'] = self.configs[module_name] + query["config"] = self.configs[module_name] response = self.misp_modules_post(query) try: - self.assertEqual(self.get_first_object_type(response), 'passive-dns') + self.assertEqual(self.get_first_object_type(response), "passive-dns") except Exception: - self.assertTrue(self.get_errors(response).startswith('There is an authentication error')) + self.assertTrue(self.get_errors(response).startswith("There is an authentication error")) else: response = self.misp_modules_post(query) - self.assertTrue(self.get_errors(response).startswith('CIRCL Passive DNS authentication is missing.')) + self.assertTrue(self.get_errors(response).startswith("CIRCL Passive DNS authentication is missing.")) def test_circl_passivessl(self): module_name = "circl_passivessl" - query = {"module": module_name, - "attribute": {"type": "ip-dst", - "value": "185.194.93.14", - "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}, - "config": {}} + query = { + "module": module_name, + "attribute": {"type": "ip-dst", "value": "185.194.93.14", "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}, + "config": {}, + } if module_name in self.configs: - query['config'] = self.configs[module_name] + query["config"] = self.configs[module_name] response = self.misp_modules_post(query) try: - self.assertEqual(self.get_first_object_type(response), 'x509') + self.assertEqual(self.get_first_object_type(response), "x509") except Exception: - self.assertTrue(self.get_errors(response).startswith('There is an authentication error')) + self.assertTrue(self.get_errors(response).startswith("There is an authentication error")) else: response = self.misp_modules_post(query) - self.assertTrue(self.get_errors(response).startswith('CIRCL Passive SSL authentication is missing.')) + self.assertTrue(self.get_errors(response).startswith("CIRCL Passive SSL authentication is missing.")) def test_countrycode(self): query = {"module": "countrycode", "domain": "www.circl.lu"} response = self.misp_modules_post(query) try: - self.assertEqual(self.get_values(response), 'Luxembourg') + self.assertEqual(self.get_values(response), "Luxembourg") except Exception: - results = ('http://www.geognos.com/api/en/countries/info/all.json not reachable', 'Unknown', - 'Not able to get the countrycode references from http://www.geognos.com/api/en/countries/info/all.json') + results = ( + "http://www.geognos.com/api/en/countries/info/all.json not reachable", + "Unknown", + "Not able to get the countrycode references from http://www.geognos.com/api/en/countries/info/all.json", + ) self.assertIn(self.get_values(response), results) def test_cve(self): @@ -202,24 +225,28 @@ def test_cve(self): "attribute": { "type": "vulnerability", "value": "CVE-2010-4444", - "uuid": "82383d84-3016-4d1c-902f-3de0533bfcec" - } + "uuid": "82383d84-3016-4d1c-902f-3de0533bfcec", + }, } response = self.misp_modules_post(query) try: - self.assertEqual(self.get_first_object_type(response), 'vulnerability') + self.assertEqual(self.get_first_object_type(response), "vulnerability") except Exception: print(self.get_errors(response)) def test_cve_advanced(self): - query = {"module": "cve_advanced", - "attribute": {"type": "vulnerability", - "value": "CVE-2010-4444", - "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}, - "config": {}} + query = { + "module": "cve_advanced", + "attribute": { + "type": "vulnerability", + "value": "CVE-2010-4444", + "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d", + }, + "config": {}, + } response = self.misp_modules_post(query) try: - self.assertEqual(self.get_first_object_type(response), 'vulnerability') + self.assertEqual(self.get_first_object_type(response), "vulnerability") except Exception: print(self.get_errors(response)) @@ -227,173 +254,176 @@ def test_dbl_spamhaus(self): query = {"module": "dbl_spamhaus", "domain": "totalmateria.net"} response = self.misp_modules_post(query) try: - self.assertEqual(self.get_values(response), 'totalmateria.net - spam test domain') + self.assertEqual(self.get_values(response), "totalmateria.net - spam test domain") except Exception: try: - self.assertTrue(self.get_values(response).startswith('The DNS query name does not exist:')) + self.assertTrue(self.get_values(response).startswith("The DNS query name does not exist:")) except Exception: - self.assertEqual(self.get_errors(response), 'Not able to reach dbl.spamhaus.org or something went wrong') + self.assertEqual( + self.get_errors(response), "Not able to reach dbl.spamhaus.org or something went wrong" + ) def test_dns(self): query = {"module": "dns", "hostname": "www.circl.lu", "config": {"nameserver": "8.8.8.8"}} response = self.misp_modules_post(query) - self.assertEqual(self.get_values(response), '185.194.93.14') + self.assertEqual(self.get_values(response), "185.194.93.14") def test_docx(self): - filename = 'test.docx' - with open(f'{self.dirname}/test_files/{filename}', 'rb') as f: + filename = "test.docx" + with open(f"{self.dirname}/test_files/{filename}", "rb") as f: encoded = b64encode(f.read()).decode() query = {"module": "docx_enrich", "attachment": filename, "data": encoded} response = self.misp_modules_post(query) - self.assertEqual(self.get_values(response), '\nThis is an basic test docx file. ') + self.assertEqual(self.get_values(response), "\nThis is an basic test docx file. ") def test_censys(self): module_name = "censys_enrich" - query = { - "attribute": {"type": "ip-dst", "value": "8.8.8.8", "uuid": ""}, - "module": module_name, - "config": {} - } + query = {"attribute": {"type": "ip-dst", "value": "8.8.8.8", "uuid": ""}, "module": module_name, "config": {}} if module_name in self.configs: - query['config'] = self.configs[module_name] + query["config"] = self.configs[module_name] response = self.misp_modules_post(query) - if self.configs[module_name].get('api_id') == '': - self.assertTrue(self.get_errors(response).startswith('ERROR: param ')) + if self.configs[module_name].get("api_id") == "": + self.assertTrue(self.get_errors(response).startswith("ERROR: param ")) else: - self.assertGreaterEqual(len(response.json().get('results', {}).get('Attribute')), 1) + self.assertGreaterEqual(len(response.json().get("results", {}).get("Attribute")), 1) else: response = self.misp_modules_post(query) - self.assertTrue(self.get_errors(response).startswith('Please provide config options')) + self.assertTrue(self.get_errors(response).startswith("Please provide config options")) def test_farsight_passivedns(self): - module_name = 'farsight_passivedns' + module_name = "farsight_passivedns" if module_name in self.configs: - query_types = ('domain', 'ip-src') - query_values = ('google.com', '8.8.8.8') - results = ('mail.casadostemperos.com.br', 'outmail.wphf.at') + query_types = ("domain", "ip-src") + query_values = ("google.com", "8.8.8.8") + results = ("mail.casadostemperos.com.br", "outmail.wphf.at") for query_type, query_value, result in zip(query_types, query_values, results): - query = {"module": module_name, query_type: query_value, 'config': self.configs[module_name]} + query = {"module": module_name, query_type: query_value, "config": self.configs[module_name]} response = self.misp_modules_post(query) try: self.assertIn(result, self.get_values(response)) except Exception: - self.assertTrue(self.get_errors(response).startswith('Something went wrong')) + self.assertTrue(self.get_errors(response).startswith("Something went wrong")) else: query = {"module": module_name, "ip-src": "8.8.8.8"} response = self.misp_modules_post(query) - self.assertEqual(self.get_errors(response), 'Farsight DNSDB apikey is missing') + self.assertEqual(self.get_errors(response), "Farsight DNSDB apikey is missing") def test_haveibeenpwned(self): - module_name = 'hibp' + module_name = "hibp" query = {"module": "hibp", "email-src": "info@circl.lu"} response = self.misp_modules_post(query) if module_name in self.configs: to_check = self.get_values(response) if to_check == "haveibeenpwned.com API not accessible (HTTP 401)": self.skipTest(f"haveibeenpwned blocks travis IPs: {response}") - self.assertEqual(to_check, 'OK (Not Found)', response) + self.assertEqual(to_check, "OK (Not Found)", response) else: - self.assertEqual(self.get_errors(response), 'Have I Been Pwned authentication is incomplete (no API key)') + self.assertEqual(self.get_errors(response), "Have I Been Pwned authentication is incomplete (no API key)") def test_hyasinsight(self): module_name = "hyasinsight" - query = {"module": module_name, - "attribute": {"type": "phone-number", - "value": "+84853620279", - "uuid": "b698dc2b-94c1-487d-8b65-3114bad5a40c"}, - "config": {}} + query = { + "module": module_name, + "attribute": { + "type": "phone-number", + "value": "+84853620279", + "uuid": "b698dc2b-94c1-487d-8b65-3114bad5a40c", + }, + "config": {}, + } if module_name in self.configs: - query['config'] = self.configs[module_name] + query["config"] = self.configs[module_name] response = self.misp_modules_post(query) - self.assertEqual(self.get_values(response)['domain'], 'tienichphongnet.com') + self.assertEqual(self.get_values(response)["domain"], "tienichphongnet.com") else: response = self.misp_modules_post(query) - self.assertEqual(self.get_errors(response), 'HYAS Insight apikey is missing') + self.assertEqual(self.get_errors(response), "HYAS Insight apikey is missing") def test_greynoise(self): - module_name = 'greynoise' + module_name = "greynoise" query = {"module": module_name, "ip-dst": "1.1.1.1"} if module_name in self.configs: - query['config'] = self.configs[module_name] + query["config"] = self.configs[module_name] response = self.misp_modules_post(query) try: - self.assertEqual(self.get_values(response), 'This IP is commonly spoofed in Internet-scan activity') + self.assertEqual(self.get_values(response), "This IP is commonly spoofed in Internet-scan activity") except Exception: self.assertIn( self.get_errors(response), - ( - "Unauthorized. Please check your API key.", - "Too many requests. You've hit the rate-limit." - ) + ("Unauthorized. Please check your API key.", "Too many requests. You've hit the rate-limit."), ) else: response = self.misp_modules_post(query) - self.assertEqual(self.get_errors(response), 'GreyNoise API Key required, but missing') + self.assertEqual(self.get_errors(response), "GreyNoise API Key required, but missing") @unittest.skip("Service doesn't work") def test_ipasn(self): - query = {"module": "ipasn", - "attribute": {"type": "ip-src", - "value": "149.13.33.14", - "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}} + query = { + "module": "ipasn", + "attribute": {"type": "ip-src", "value": "149.13.33.14", "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}, + } response = self.misp_modules_post(query) - self.assertEqual(self.get_first_object_type(response), 'asn') + self.assertEqual(self.get_first_object_type(response), "asn") def test_ipqs_fraud_and_risk_scoring(self): module_name = "ipqs_fraud_and_risk_scoring" - query = {"module": module_name, - "attribute": {"type": "email", - "value": "noreply@ipqualityscore.com", - "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}, - "config": {}} + query = { + "module": module_name, + "attribute": { + "type": "email", + "value": "noreply@ipqualityscore.com", + "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d", + }, + "config": {}, + } if module_name in self.configs: - query['config'] = self.configs[module_name] + query["config"] = self.configs[module_name] response = self.misp_modules_post(query) - self.assertEqual(self.get_values(response)['message'], 'Success.') + self.assertEqual(self.get_values(response)["message"], "Success.") else: response = self.misp_modules_post(query) - self.assertEqual(self.get_errors(response), 'IPQualityScore apikey is missing') + self.assertEqual(self.get_errors(response), "IPQualityScore apikey is missing") def test_macaddess_io(self): - module_name = 'macaddress_io' + module_name = "macaddress_io" query = {"module": module_name, "mac-address": "44:38:39:ff:ef:57"} if module_name in self.configs: query["config"] = self.configs[module_name] response = self.misp_modules_post(query) - self.assertEqual(self.get_values(response)['Valid MAC address'], 'True') + self.assertEqual(self.get_values(response)["Valid MAC address"], "True") else: response = self.misp_modules_post(query) - self.assertEqual(self.get_errors(response), 'Authorization required') + self.assertEqual(self.get_errors(response), "Authorization required") def test_macvendors(self): query = {"module": "macvendors", "mac-address": "FC-A1-3E-2A-1C-33"} response = self.misp_modules_post(query) - self.assertEqual(self.get_values(response), 'Samsung Electronics Co.,Ltd') + self.assertEqual(self.get_values(response), "Samsung Electronics Co.,Ltd") def test_ocr(self): - filename = 'misp-logo.png' - with open(f'{self.dirname}/test_files/{filename}', 'rb') as f: + filename = "misp-logo.png" + with open(f"{self.dirname}/test_files/{filename}", "rb") as f: encoded = b64encode(f.read()).decode() query = {"module": "ocr_enrich", "attachment": filename, "data": encoded} response = self.misp_modules_post(query) - self.assertEqual(self.get_values(response).strip('\n'), 'Threat Sharing') + self.assertEqual(self.get_values(response).strip("\n"), "Threat Sharing") def test_ods(self): - filename = 'test.ods' - with open(f'{self.dirname}/test_files/{filename}', 'rb') as f: + filename = "test.ods" + with open(f"{self.dirname}/test_files/{filename}", "rb") as f: encoded = b64encode(f.read()).decode() query = {"module": "ods_enrich", "attachment": filename, "data": encoded} response = self.misp_modules_post(query) - self.assertEqual(self.get_values(response), '\n column.0\n0 ods test') + self.assertEqual(self.get_values(response), "\n column.0\n0 ods test") def test_odt(self): - filename = 'test.odt' - with open(f'{self.dirname}/test_files/{filename}', 'rb') as f: + filename = "test.odt" + with open(f"{self.dirname}/test_files/{filename}", "rb") as f: encoded = b64encode(f.read()).decode() query = {"module": "odt_enrich", "attachment": filename, "data": encoded} response = self.misp_modules_post(query) - self.assertEqual(self.get_values(response), 'odt test') + self.assertEqual(self.get_values(response), "odt test") def test_onyphe(self): module_name = "onyphe" @@ -404,12 +434,12 @@ def test_onyphe(self): query["config"] = self.configs[module_name] response = self.misp_modules_post(query) try: - self.assertTrue(self.get_values(response).startswith('https://pastebin.com/raw/')) + self.assertTrue(self.get_values(response).startswith("https://pastebin.com/raw/")) except Exception: - self.assertEqual(self.get_errors(response), 'no more credits') + self.assertEqual(self.get_errors(response), "no more credits") else: response = self.misp_modules_post(query) - self.assertEqual(self.get_errors(response), 'Onyphe authentication is missing') + self.assertEqual(self.get_errors(response), "Onyphe authentication is missing") def test_onyphe_full(self): module_name = "onyphe_full" @@ -420,20 +450,22 @@ def test_onyphe_full(self): query["config"] = self.configs[module_name] response = self.misp_modules_post(query) try: - self.assertEqual(self.get_values(response), '37.7510,-97.8220') + self.assertEqual(self.get_values(response), "37.7510,-97.8220") except Exception: - self.assertTrue(self.get_errors(response).startswith('Error ')) + self.assertTrue(self.get_errors(response).startswith("Error ")) else: response = self.misp_modules_post(query) - self.assertEqual(self.get_errors(response), 'Onyphe authentication is missing') + self.assertEqual(self.get_errors(response), "Onyphe authentication is missing") @unittest.skip("Unreliable results") def test_otx(self): - query_types = ('domain', 'ip-src', 'md5') - query_values = ('circl.lu', '8.8.8.8', '616eff3e9a7575ae73821b4668d2801c') - results = (('149.13.33.14', '149.13.33.17', '6f9814ba70e68c3bce16d253e8d8f86e04a21a2b4172a0f7631040096ba2c47a'), - 'ffc2595aefa80b61621023252b5f0ccb22b6e31d7f1640913cd8ff74ddbd8b41', - '8.8.8.8') + query_types = ("domain", "ip-src", "md5") + query_values = ("circl.lu", "8.8.8.8", "616eff3e9a7575ae73821b4668d2801c") + results = ( + ("149.13.33.14", "149.13.33.17", "6f9814ba70e68c3bce16d253e8d8f86e04a21a2b4172a0f7631040096ba2c47a"), + "ffc2595aefa80b61621023252b5f0ccb22b6e31d7f1640913cd8ff74ddbd8b41", + "8.8.8.8", + ) for query_type, query_value, result in zip(query_types, query_values, results): query = {"module": "otx", query_type: query_value, "config": {"apikey": "1"}} response = self.misp_modules_post(query) @@ -450,43 +482,47 @@ def test_passivetotal(self): query["config"] = self.configs[module_name] response = self.misp_modules_post(query) try: - self.assertIn('www.circl.lu', response.json()['results'][0]['values']) + self.assertIn("www.circl.lu", response.json()["results"][0]["values"]) except Exception: - self.assertIn(self.get_errors(response), ('We hit an error, time to bail!', 'API quota exceeded.')) + self.assertIn(self.get_errors(response), ("We hit an error, time to bail!", "API quota exceeded.")) else: response = self.misp_modules_post(query) - self.assertEqual(self.get_errors(response), 'Configuration is missing from the request.') + self.assertEqual(self.get_errors(response), "Configuration is missing from the request.") def test_pdf(self): - filename = 'test.pdf' - with open(f'{self.dirname}/test_files/{filename}', 'rb') as f: + filename = "test.pdf" + with open(f"{self.dirname}/test_files/{filename}", "rb") as f: encoded = b64encode(f.read()).decode() query = {"module": "pdf_enrich", "attachment": filename, "data": encoded} response = self.misp_modules_post(query) - self.assertRegex(self.get_values(response), r'^Pdf test') + self.assertRegex(self.get_values(response), r"^Pdf test") def test_pptx(self): - filename = 'test.pptx' - with open(f'{self.dirname}/test_files/{filename}', 'rb') as f: + filename = "test.pptx" + with open(f"{self.dirname}/test_files/{filename}", "rb") as f: encoded = b64encode(f.read()).decode() query = {"module": "pptx_enrich", "attachment": filename, "data": encoded} response = self.misp_modules_post(query) - self.assertEqual(self.get_values(response), '\npptx test\n') + self.assertEqual(self.get_values(response), "\npptx test\n") def test_qrcode(self): - filename = 'qrcode.jpeg' - with open(f'{self.dirname}/test_files/{filename}', 'rb') as f: + filename = "qrcode.jpeg" + with open(f"{self.dirname}/test_files/{filename}", "rb") as f: encoded = b64encode(f.read()).decode() query = {"module": "qrcode", "attachment": filename, "data": encoded} response = self.misp_modules_post(query) - self.assertEqual(self.get_values(response), '1GXZ6v7FZzYBEnoRaG77SJxhu7QkvQmFuh') + self.assertEqual(self.get_values(response), "1GXZ6v7FZzYBEnoRaG77SJxhu7QkvQmFuh") def test_ransomcoindb(self): - query = {"module": "ransomcoindb", - "attributes": {"type": "btc", - "value": "1ES14c7qLb5CYhLMUekctxLgc1FV2Ti9DA", - "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}} - if 'ransomcoindb' not in self.configs: + query = { + "module": "ransomcoindb", + "attributes": { + "type": "btc", + "value": "1ES14c7qLb5CYhLMUekctxLgc1FV2Ti9DA", + "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d", + }, + } + if "ransomcoindb" not in self.configs: response = self.misp_modules_post(query) self.assertEqual(self.get_errors(response), "Ransomcoindb API key is missing") @@ -496,20 +532,20 @@ def test_rbl(self): query = {"module": "rbl", "ip-src": "8.8.8.8"} response = self.misp_modules_post(query) try: - self.assertTrue(self.get_values(response).startswith('8.8.8.8.bl.spamcannibal.org')) + self.assertTrue(self.get_values(response).startswith("8.8.8.8.bl.spamcannibal.org")) except Exception: self.assertEqual(self.get_errors(response), "No data found by querying known RBLs") def test_reversedns(self): query = {"module": "reversedns", "ip-src": "8.8.8.8"} response = self.misp_modules_post(query) - self.assertEqual(self.get_values(response), 'dns.google.') + self.assertEqual(self.get_values(response), "dns.google.") def test_securitytrails(self): module_name = "securitytrails" - query_types = ('ip-src', 'domain') - query_values = ('149.13.33.14', 'circl.lu') - results = ('circl.lu', 'ns4.eurodns.com') + query_types = ("ip-src", "domain") + query_values = ("149.13.33.14", "circl.lu") + results = ("circl.lu", "ns4.eurodns.com") if module_name in self.configs: for query_type, query_value, result in zip(query_types, query_values, results): query = {"module": module_name, query_type: query_value, "config": self.configs[module_name]} @@ -517,39 +553,39 @@ def test_securitytrails(self): try: self.assertEqual(self.get_values(response), result) except Exception: - self.assertTrue(self.get_errors(response).startswith("You've exceeded the usage limits for your account.")) + self.assertTrue( + self.get_errors(response).startswith("You've exceeded the usage limits for your account.") + ) else: query = {"module": module_name, query_values[0]: query_types[0]} response = self.misp_modules_post(query) - self.assertEqual(self.get_errors(response), 'SecurityTrails authentication is missing') + self.assertEqual(self.get_errors(response), "SecurityTrails authentication is missing") def test_shodan(self): module_name = "shodan" query = { "module": module_name, - "attribute": { - "uuid": "a21aae0c-7426-4762-9b79-854314d69059", - "type": "ip-src", - "value": "149.13.33.14" - } + "attribute": {"uuid": "a21aae0c-7426-4762-9b79-854314d69059", "type": "ip-src", "value": "149.13.33.14"}, } if module_name in self.configs: - query['config'] = self.configs[module_name] + query["config"] = self.configs[module_name] response = self.misp_modules_post(query) - self.assertEqual(self.get_first_object_type(response), 'ip-api-address') + self.assertEqual(self.get_first_object_type(response), "ip-api-address") else: response = self.misp_modules_post(query) - self.assertEqual(self.get_errors(response), 'Shodan authentication is missing') + self.assertEqual(self.get_errors(response), "Shodan authentication is missing") def test_sigma_queries(self): query = {"module": "sigma_queries", "sigma": self.sigma_rule} response = self.misp_modules_post(query) - self.assertTrue(self.get_values(response)['kibana'].startswith('[\n {\n "_id": "Antivirus-Web-Shell-Detection"')) + self.assertTrue( + self.get_values(response)["kibana"].startswith('[\n {\n "_id": "Antivirus-Web-Shell-Detection"') + ) def test_sigma_syntax(self): query = {"module": "sigma_syntax_validator", "sigma": self.sigma_rule} response = self.misp_modules_post(query) - self.assertTrue(self.get_values(response).startswith('Syntax valid:')) + self.assertTrue(self.get_values(response).startswith("Syntax valid:")) def test_sourcecache(self): input_value = "https://www.misp-project.org/feeds/" @@ -561,14 +597,14 @@ def test_sourcecache(self): def test_stix2_pattern_validator(self): query = {"module": "stix2_pattern_syntax_validator", "stix2-pattern": "[ipv4-addr:value = '8.8.8.8']"} response = self.misp_modules_post(query) - self.assertEqual(self.get_values(response), 'Syntax valid') + self.assertEqual(self.get_values(response), "Syntax valid") def test_threatcrowd(self): if LiveCI: return True - query_types = ('domain', 'ip-src', 'md5', 'whois-registrant-email') - query_values = ('circl.lu', '149.13.33.14', '616eff3e9a7575ae73821b4668d2801c', 'hostmaster@eurodns.com') - results = ('149.13.33.4', 'cve.circl.lu', 'devilreturns.com', 'navabi.lu') + query_types = ("domain", "ip-src", "md5", "whois-registrant-email") + query_values = ("circl.lu", "149.13.33.14", "616eff3e9a7575ae73821b4668d2801c", "hostmaster@eurodns.com") + results = ("149.13.33.4", "cve.circl.lu", "devilreturns.com", "navabi.lu") for query_type, query_value, result in zip(query_types, query_values, results): query = {"module": "threatcrowd", query_type: query_value} response = self.misp_modules_post(query) @@ -576,29 +612,25 @@ def test_threatcrowd(self): def test_crowdstrike(self): module_name = "crowdstrike_falcon" - query = { - "attribute": {"type": "sha256", "value": "", "uuid": ""}, - "module": module_name, - "config": {} - } + query = {"attribute": {"type": "sha256", "value": "", "uuid": ""}, "module": module_name, "config": {}} if module_name in self.configs: - query['config'] = self.configs[module_name] + query["config"] = self.configs[module_name] response = self.misp_modules_post(query) - if self.configs[module_name].get('api_id') == '': - self.assertTrue(self.get_errors(response).startswith('HTTP Error:')) + if self.configs[module_name].get("api_id") == "": + self.assertTrue(self.get_errors(response).startswith("HTTP Error:")) else: - self.assertGreaterEqual(len(response.json().get('results', {}).get('Attribute')), 1) + self.assertGreaterEqual(len(response.json().get("results", {}).get("Attribute")), 1) else: response = self.misp_modules_post(query) - self.assertTrue(self.get_errors(response).startswith('CrowdStrike apikey is missing')) + self.assertTrue(self.get_errors(response).startswith("CrowdStrike apikey is missing")) def test_threatminer(self): if LiveCI: return True - query_types = ('domain', 'ip-src', 'md5') - query_values = ('circl.lu', '149.13.33.4', 'b538dbc6160ef54f755a540e06dc27cd980fc4a12005e90b3627febb44a1a90f') - results = ('149.13.33.14', 'f6ecb9d5c21defb1f622364a30cb8274f817a1a2', 'http://www.circl.lu/') + query_types = ("domain", "ip-src", "md5") + query_values = ("circl.lu", "149.13.33.4", "b538dbc6160ef54f755a540e06dc27cd980fc4a12005e90b3627febb44a1a90f") + results = ("149.13.33.14", "f6ecb9d5c21defb1f622364a30cb8274f817a1a2", "http://www.circl.lu/") for query_type, query_value, result in zip(query_types, query_values, results): query = {"module": "threatminer", query_type: query_value} response = self.misp_modules_post(query) @@ -606,26 +638,29 @@ def test_threatminer(self): @unittest.skip("Service doesn't work") def test_urlhaus(self): - query_types = ('domain', 'ip-src', 'sha256', 'url') - query_values = ('www.bestwpdesign.com', '79.118.195.239', - 'a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3', - 'http://79.118.195.239:1924/.i') - results = ('url', 'url', 'file', 'virustotal-report') + query_types = ("domain", "ip-src", "sha256", "url") + query_values = ( + "www.bestwpdesign.com", + "79.118.195.239", + "a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3", + "http://79.118.195.239:1924/.i", + ) + results = ("url", "url", "file", "virustotal-report") for query_type, query_value, result in zip(query_types[:2], query_values[:2], results[:2]): - query = {"module": "urlhaus", - "attribute": {"type": query_type, - "value": query_value, - "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}} + query = { + "module": "urlhaus", + "attribute": {"type": query_type, "value": query_value, "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}, + } response = self.misp_modules_post(query) print(response.json()) self.assertIn(result, self.get_attribute_types(response)) for query_type, query_value, result in zip(query_types[2:], query_values[2:], results[2:]): - query = {"module": "urlhaus", - "attribute": {"type": query_type, - "value": query_value, - "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}} + query = { + "module": "urlhaus", + "attribute": {"type": query_type, "value": query_value, "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}, + } response = self.misp_modules_post(query) print(response.json()) self.assertIn(result, self.get_object_types(response)) @@ -634,96 +669,62 @@ def test_urlscan(self): module_name = "urlscan" query = {"module": module_name, "url": "https://circl.lu/team"} if module_name in self.configs: - query['config'] = self.configs[module_name] + query["config"] = self.configs[module_name] response = self.misp_modules_post(query) - self.assertEqual(self.get_values(response), 'circl.lu') + self.assertEqual(self.get_values(response), "circl.lu") else: response = self.misp_modules_post(query) - self.assertEqual(self.get_errors(response), 'Urlscan apikey is missing') + self.assertEqual(self.get_errors(response), "Urlscan apikey is missing") def test_virustotal_public(self): module_name = "virustotal_public" attributes = ( - { - "uuid": "ffea0594-355a-42fe-9b98-fad28fd248b3", - "type": "domain", - "value": "circl.lu" - }, - { - "uuid": "1f3f0f2d-5143-4b05-a0f1-8ac82f51a979", - "type": "ip-src", - "value": "149.13.33.14" - }, + {"uuid": "ffea0594-355a-42fe-9b98-fad28fd248b3", "type": "domain", "value": "circl.lu"}, + {"uuid": "1f3f0f2d-5143-4b05-a0f1-8ac82f51a979", "type": "ip-src", "value": "149.13.33.14"}, { "uuid": "b4be6652-f4ff-4515-ae63-3f016df37e8f", "type": "sha256", - "value": "a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3" + "value": "a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3", }, - { - "uuid": "6cead544-b683-48cb-b19b-a2561ffa1f51", - "type": "url", - "value": "http://194.169.88.56:49151/.i" - } + {"uuid": "6cead544-b683-48cb-b19b-a2561ffa1f51", "type": "url", "value": "http://194.169.88.56:49151/.i"}, ) - results = ('whois', 'asn', 'file', 'virustotal-report') + results = ("whois", "asn", "file", "virustotal-report") if module_name in self.configs: for attribute, result in zip(attributes, results): - query = {"module": module_name, - "attribute": attribute, - "config": self.configs[module_name]} + query = {"module": module_name, "attribute": attribute, "config": self.configs[module_name]} response = self.misp_modules_post(query) try: self.assertEqual(self.get_first_object_type(response), result) except Exception: self.assertEqual(self.get_errors(response), "VirusTotal request rate limit exceeded.") else: - query = { - "module": module_name, - "attribute": attributes[0] - } + query = {"module": module_name, "attribute": attributes[0]} response = self.misp_modules_post(query) self.assertEqual(self.get_errors(response), "A VirusTotal api key is required for this module.") def test_virustotal(self): module_name = "virustotal" attributes = ( - { - "uuid": "ffea0594-355a-42fe-9b98-fad28fd248b3", - "type": "domain", - "value": "circl.lu" - }, - { - "uuid": "1f3f0f2d-5143-4b05-a0f1-8ac82f51a979", - "type": "ip-src", - "value": "149.13.33.14" - }, + {"uuid": "ffea0594-355a-42fe-9b98-fad28fd248b3", "type": "domain", "value": "circl.lu"}, + {"uuid": "1f3f0f2d-5143-4b05-a0f1-8ac82f51a979", "type": "ip-src", "value": "149.13.33.14"}, { "uuid": "b4be6652-f4ff-4515-ae63-3f016df37e8f", "type": "sha256", - "value": "a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3" + "value": "a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3", }, - { - "uuid": "6cead544-b683-48cb-b19b-a2561ffa1f51", - "type": "url", - "value": "http://194.169.88.56:49151/.i" - } + {"uuid": "6cead544-b683-48cb-b19b-a2561ffa1f51", "type": "url", "value": "http://194.169.88.56:49151/.i"}, ) - results = ('domain-ip', 'asn', 'virustotal-report', 'virustotal-report') + results = ("domain-ip", "asn", "virustotal-report", "virustotal-report") if module_name in self.configs: for attribute, result in zip(attributes, results): - query = {"module": module_name, - "attribute": attribute, - "config": self.configs[module_name]} + query = {"module": module_name, "attribute": attribute, "config": self.configs[module_name]} response = self.misp_modules_post(query) try: self.assertEqual(self.get_first_object_type(response), result) except Exception: self.assertEqual(self.get_errors(response), "VirusTotal request rate limit exceeded.") else: - query = { - "module": module_name, - "attribute": attributes[0] - } + query = {"module": module_name, "attribute": attributes[0]} response = self.misp_modules_post(query) self.assertEqual(self.get_errors(response), "A VirusTotal api key is required for this module.") @@ -731,7 +732,7 @@ def test_vulners(self): module_name = "vulners" query = {"module": module_name, "vulnerability": "CVE-2010-3333"} if module_name in self.configs: - query['config'] = self.configs[module_name] + query["config"] = self.configs[module_name] response = self.misp_modules_post(query) self.assertTrue(self.get_values(response).endswith('"RTF Stack Buffer Overflow Vulnerability."')) else: @@ -742,53 +743,75 @@ def test_wikidata(self): query = {"module": "wiki", "text": "Google"} response = self.misp_modules_post(query) try: - self.assertEqual(self.get_values(response), 'http://www.wikidata.org/entity/Q95') + self.assertEqual(self.get_values(response), "http://www.wikidata.org/entity/Q95") except KeyError: - self.assertEqual(self.get_errors(response), 'Something went wrong, look in the server logs for details') + self.assertEqual(self.get_errors(response), "Something went wrong, look in the server logs for details") except Exception: - self.assertEqual(self.get_values(response), 'No additional data found on Wikidata') + self.assertEqual(self.get_values(response), "No additional data found on Wikidata") def test_xforceexchange(self): module_name = "xforceexchange" - query_types = ('domain', 'ip-src', 'md5', 'url', 'vulnerability') - query_values = ('mediaget.com', '61.255.239.86', '474b9ccf5ab9d72ca8a333889bbb34f0', - 'mediaget.com', 'CVE-2014-2601') - results = ('domain-ip', 'domain-ip', 'url', 'domain-ip', 'vulnerability') + query_types = ("domain", "ip-src", "md5", "url", "vulnerability") + query_values = ( + "mediaget.com", + "61.255.239.86", + "474b9ccf5ab9d72ca8a333889bbb34f0", + "mediaget.com", + "CVE-2014-2601", + ) + results = ("domain-ip", "domain-ip", "url", "domain-ip", "vulnerability") if module_name in self.configs: for query_type, query_value, result in zip(query_types, query_values, results): - query = {"module": module_name, - "attribute": {"type": query_type, - "value": query_value, - "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}, - "config": self.configs[module_name]} + query = { + "module": module_name, + "attribute": { + "type": query_type, + "value": query_value, + "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d", + }, + "config": self.configs[module_name], + } response = self.misp_modules_post(query) self.assertEqual(self.get_first_object_type(response), result) else: - query = {"module": module_name, - "attribute": {"type": query_types[0], - "value": query_values[0], - "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}} + query = { + "module": module_name, + "attribute": { + "type": query_types[0], + "value": query_values[0], + "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d", + }, + } response = self.misp_modules_post(query) self.assertEqual(self.get_errors(response), "An API authentication is required (key and password).") def test_xlsx(self): if LiveCI: return True - filename = 'test.xlsx' - with open(f'{self.dirname}/test_files/{filename}', 'rb') as f: + filename = "test.xlsx" + with open(f"{self.dirname}/test_files/{filename}", "rb") as f: encoded = b64encode(f.read()).decode() query = {"module": "xlsx_enrich", "attachment": filename, "data": encoded} response = self.misp_modules_post(query) - self.assertEqual(self.get_values(response), ' header\n0 xlsx test') + self.assertEqual(self.get_values(response), " header\n0 xlsx test") def test_yara_query(self): query = {"module": "yara_query", "md5": "b2a5abfeef9e36964281a31e17b57c97"} response = self.misp_modules_post(query) - expected_result = 'import "hash"\r\nrule MD5 {\r\n\tcondition:\r\n\t\thash.md5(0, filesize) == "b2a5abfeef9e36964281a31e17b57c97"\r\n}' + expected_result = ( + 'import "hash"\r\nrule MD5 {\r\n\tcondition:\r\n\t\thash.md5(0, filesize) ==' + ' "b2a5abfeef9e36964281a31e17b57c97"\r\n}' + ) self.assertEqual(self.get_values(response), expected_result) def test_yara_validator(self): - query = {"module": "yara_syntax_validator", "yara": 'import "hash"\r\nrule MD5 {\r\n\tcondition:\r\n\t\thash.md5(0, filesize) == "b2a5abfeef9e36964281a31e17b57c97"\r\n}'} + query = { + "module": "yara_syntax_validator", + "yara": ( + 'import "hash"\r\nrule MD5 {\r\n\tcondition:\r\n\t\thash.md5(0, filesize) ==' + ' "b2a5abfeef9e36964281a31e17b57c97"\r\n}' + ), + } response = self.misp_modules_post(query) - self.assertEqual(self.get_values(response), 'Syntax valid') + self.assertEqual(self.get_values(response), "Syntax valid") diff --git a/tests/test_exports.py b/tests/test_exports.py index 785123be9..a8efb942b 100644 --- a/tests/test_exports.py +++ b/tests/test_exports.py @@ -4,14 +4,16 @@ import json import os import unittest -import requests from urllib.parse import urljoin +import requests + class TestExports(unittest.TestCase): """Unittest module for export modules""" + def setUp(self): - self.headers = {'Content-Type': 'application/json'} + self.headers = {"Content-Type": "application/json"} self.url = "http://127.0.0.1:6666/" input_event_path = "%s/test_files/misp_event.json" % os.path.dirname(os.path.realpath(__file__)) with open(input_event_path, "r") as ifile: @@ -23,8 +25,8 @@ def misp_modules_post(self, query): @staticmethod def get_values(response): data = response.json() - if 'data' in data: - return base64.b64decode(data['data']).decode("utf-8") + if "data" in data: + return base64.b64decode(data["data"]).decode("utf-8") def test_introspection(self): """checks if all export modules are offered through the misp-modules service""" @@ -32,8 +34,15 @@ def test_introspection(self): response = requests.get(self.url + "modules") modules = [module["name"] for module in response.json()] # list modules in the export_mod folder - export_mod_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'misp_modules', 'modules', "export_mod") - module_files = [file[:-3] for file in os.listdir(export_mod_path) if file.endswith(".py") if file not in ['__init__.py', 'testexport.py']] + export_mod_path = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "..", "misp_modules", "modules", "export_mod" + ) + module_files = [ + file[:-3] + for file in os.listdir(export_mod_path) + if file.endswith(".py") + if file not in ["__init__.py", "testexport.py"] + ] missing = [] for module in module_files: if module not in modules: @@ -45,13 +54,7 @@ def test_introspection(self): def test_threat_connect_export(self): """Test an event export""" test_source = "Test Export" - query = { - "module": 'threat_connect_export', - "data": [self.event], - "config": { - "Default_Source": test_source - } - } + query = {"module": "threat_connect_export", "data": [self.event], "config": {"Default_Source": test_source}} try: response = self.misp_modules_post(query) @@ -77,7 +80,12 @@ def test_yara_export(self): "data": [self.event], } response = self.misp_modules_post(query) - expected_result = 'rule MISP_e625_MetadataExample\n{\n meta:\n my_identifier_1 = "Some string data"\n my_identifier_2 = 24\n my_identifier_3 = true\n\n strings:\n $my_text_string = "text here"\n $my_hex_string = { E2 34 A1 C8 23 FB }\n\n condition:\n $my_text_string or $my_hex_string\n}\n\n' + expected_result = ( + 'rule MISP_e625_MetadataExample\n{\n meta:\n my_identifier_1 = "Some string data"\n ' + ' my_identifier_2 = 24\n my_identifier_3 = true\n\n strings:\n $my_text_string = "text' + ' here"\n $my_hex_string = { E2 34 A1 C8 23 FB }\n\n condition:\n $my_text_string or' + " $my_hex_string\n}\n\n" + ) result = self.get_values(response) self.assertEqual(result, expected_result) diff --git a/tests/test_imports.py b/tests/test_imports.py index 725e4e78b..bad1d7f23 100644 --- a/tests/test_imports.py +++ b/tests/test_imports.py @@ -1,12 +1,14 @@ import os import unittest + import requests class TestImports(unittest.TestCase): """Unittest module for import modules""" + def setUp(self): - self.headers = {'Content-Type': 'application/json'} + self.headers = {"Content-Type": "application/json"} self.url = "http://127.0.0.1:6666/" def test_introspection(self): @@ -15,8 +17,15 @@ def test_introspection(self): response = requests.get(self.url + "modules") modules = [module["name"] for module in response.json()] # list modules in the export_mod folder - export_mod_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'misp_modules', 'modules', "import_mod") - module_files = [file[:-3] for file in os.listdir(export_mod_path) if file.endswith(".py") if file not in ['__init__.py', 'testimport.py']] + export_mod_path = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "..", "misp_modules", "modules", "import_mod" + ) + module_files = [ + file[:-3] + for file in os.listdir(export_mod_path) + if file.endswith(".py") + if file not in ["__init__.py", "testimport.py"] + ] missing = [] for module in module_files: if module not in modules: diff --git a/tests/test_yara.py b/tests/test_yara.py index 4d0bf64f2..89da88b74 100644 --- a/tests/test_yara.py +++ b/tests/test_yara.py @@ -1,8 +1,8 @@ - import json import os -import unittest import sys +import unittest + try: import yara except (OSError, ImportError): @@ -11,8 +11,9 @@ class TestYara(unittest.TestCase): """Unittest module for yara related modules""" + def setUp(self): - self.headers = {'Content-Type': 'application/json'} + self.headers = {"Content-Type": "application/json"} self.url = "http://127.0.0.1:6666/" self.module = "threat_connect_export" input_event_path = "%s/test_files/misp_event.json" % os.path.dirname(os.path.realpath(__file__)) @@ -20,7 +21,7 @@ def setUp(self): self.event = json.load(ifile) def test_install(self): - files = ['tests/yara_hash_module_test.yara', 'tests/yara_pe_module_test.yara'] + files = ["tests/yara_hash_module_test.yara", "tests/yara_pe_module_test.yara"] for file_ in files: try: