Merge remote-tracking branch 'origin/main'

MISP · Feb 24, 2025 · 93d8e35 · 93d8e35
2 parents 44a915f + 625bc68
commit 93d8e35
Show file tree

Hide file tree

Showing 15 changed files with 908 additions and 2,663 deletions.
diff --git a/README.md b/README.md
@@ -99,6 +99,7 @@ For further Information see the [license file](https://misp.github.io/misp-modul
 * [OCR Enrich](https://misp.github.io/misp-modules/expansion/#ocr-enrich) - Module to process some optical character recognition on pictures.
 * [ODS Enrich](https://misp.github.io/misp-modules/expansion/#ods-enrich) - Module to extract freetext from a .ods document.
 * [ODT Enrich](https://misp.github.io/misp-modules/expansion/#odt-enrich) - Module to extract freetext from a .odt document.
+* [Onion Lookup](https://misp.github.io/misp-modules/expansion/#onion-lookup) - MISP module using the MISP standard. Uses the onion-lookup service to get information about an onion.
 * [Onyphe Lookup](https://misp.github.io/misp-modules/expansion/#onyphe-lookup) - Module to process a query on Onyphe.
 * [Onyphe Full Lookup](https://misp.github.io/misp-modules/expansion/#onyphe-full-lookup) - Module to process a full query on Onyphe.
 * [AlienVault OTX Lookup](https://misp.github.io/misp-modules/expansion/#alienvault-otx-lookup) - Module to get information from AlienVault OTX.

diff --git a/documentation/README.md b/documentation/README.md
@@ -1740,6 +1740,18 @@ Module to extract freetext from a .odt document.
 
 -----
 
+#### [Onion Lookup](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/onion_lookup.py)
+
+<img src=logos/onion.png height=60>
+
+MISP module using the MISP standard. Uses the onion-lookup service to get information about an onion.
+[[source code](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/onion_lookup.py)]
+
+- **references**:
+>https://onion.ail-project.org/
+
+-----
+
 #### [Onyphe Lookup](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/onyphe.py)
 
 <img src=logos/onyphe.jpg height=60>

diff --git a/documentation/logos/onion.png b/documentation/logos/onion.png
diff --git a/documentation/mkdocs/expansion.md b/documentation/mkdocs/expansion.md
@@ -1737,6 +1737,18 @@ Module to extract freetext from a .odt document.
 
 -----
 
+#### [Onion Lookup](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/onion_lookup.py)
+
+<img src=../logos/onion.png height=60>
+
+MISP module using the MISP standard. Uses the onion-lookup service to get information about an onion.
+[[source code](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/onion_lookup.py)]
+
+- **references**:
+>https://onion.ail-project.org/
+
+-----
+
 #### [Onyphe Lookup](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/onyphe.py)
 
 <img src=../logos/onyphe.jpg height=60>

diff --git a/documentation/mkdocs/index.md b/documentation/mkdocs/index.md
@@ -78,6 +78,7 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj
 * [OCR Enrich](https://misp.github.io/misp-modules/expansion/#ocr-enrich) - Module to process some optical character recognition on pictures.
 * [ODS Enrich](https://misp.github.io/misp-modules/expansion/#ods-enrich) - Module to extract freetext from a .ods document.
 * [ODT Enrich](https://misp.github.io/misp-modules/expansion/#odt-enrich) - Module to extract freetext from a .odt document.
+* [Onion Lookup](https://misp.github.io/misp-modules/expansion/#onion-lookup) - MISP module using the MISP standard. Uses the onion-lookup service to get information about an onion.
 * [Onyphe Lookup](https://misp.github.io/misp-modules/expansion/#onyphe-lookup) - Module to process a query on Onyphe.
 * [Onyphe Full Lookup](https://misp.github.io/misp-modules/expansion/#onyphe-full-lookup) - Module to process a full query on Onyphe.
 * [AlienVault OTX Lookup](https://misp.github.io/misp-modules/expansion/#alienvault-otx-lookup) - Module to get information from AlienVault OTX.

diff --git a/misp_modules/lib/stix2misp.py b/misp_modules/lib/stix2misp.py
diff --git a/misp_modules/lib/stix2misp_mapping.py b/misp_modules/lib/stix2misp_mapping.py
diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py
@@ -3,27 +3,120 @@
 
 sys.path.append('{}/lib'.format('/'.join((os.path.realpath(__file__)).split('/')[:-3])))
 
-__all__ = ['cuckoo_submit', 'vmray_submit', 'circl_passivedns', 'circl_passivessl',
-           'cluster25_expand', 'countrycode', 'cve', 'cve_advanced', 'cpe', 'dns', 'btc_steroids', 'domaintools',
-           'eupi', 'eql', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal',
-           'shodan', 'reversedns', 'geoip_asn', 'geoip_city', 'geoip_country', 'wiki', 'iprep',
-           'threatminer', 'otx', 'threatcrowd', 'vulndb', 'crowdstrike_falcon',
-           'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl',
-           'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator',
-           'sigma_queries', 'dbl_spamhaus', 'vulners', 'yara_query', 'macaddress_io',
-           'intel471', 'backscatter_io', 'btc_scam_check', 'hibp', 'greynoise', 'macvendors',
-           'qrcode', 'ocr_enrich', 'pdf_enrich', 'docx_enrich', 'xlsx_enrich', 'pptx_enrich',
-           'ods_enrich', 'odt_enrich', 'joesandbox_submit', 'joesandbox_query', 'urlhaus',
-           'virustotal_public', 'apiosintds', 'urlscan', 'securitytrails', 'apivoid',
-           'assemblyline_submit', 'assemblyline_query', 'ransomcoindb', 'malwarebazaar',
-           'lastline_query', 'lastline_submit', 'sophoslabs_intelix', 'cytomic_orion', 'censys_enrich',
-           'trustar_enrich', 'recordedfuture', 'html_to_markdown', 'socialscan', 'passive_ssh',
-           'qintel_qsentry', 'mwdb', 'hashlookup', 'mmdb_lookup', 'ipqs_fraud_and_risk_scoring',
-           'clamav', 'jinja_template_rendering', 'hyasinsight', 'variotdbs', 'crowdsec',
-           'extract_url_components', 'ipinfo', 'whoisfreaks', 'ip2locationio', 'stairwell',
-           'google_threat_intelligence', 'vulnerability_lookup', 'vysion', 'mcafee_insights_enrich',
-           'threatfox', 'yeti', 'abuseipdb', 'vmware_nsx', 'sigmf_expand', 'google_safe_browsing',
-           'google_search', 'whois', 'triage_submit', 'virustotal_upload', 'malshare_upload', 'convert_markdown_to_pdf' ]
+__all__ = [
+    'cuckoo_submit',
+    'vmray_submit',
+    'circl_passivedns',
+    'circl_passivessl',
+    'cluster25_expand',
+    'countrycode',
+    'cve',
+    'cve_advanced',
+    'cpe',
+    'dns',
+    'btc_steroids',
+    'domaintools',
+    'eupi',
+    'eql',
+    'farsight_passivedns',
+    'ipasn',
+    'passivetotal',
+    'sourcecache',
+    'virustotal',
+    'shodan',
+    'reversedns',
+    'geoip_asn',
+    'geoip_city',
+    'geoip_country',
+    'wiki',
+    'iprep',
+    'threatminer',
+    'otx',
+    'threatcrowd',
+    'vulndb',
+    'crowdstrike_falcon',
+    'yara_syntax_validator',
+    'hashdd',
+    'onyphe',
+    'onyphe_full',
+    'rbl',
+    'xforceexchange',
+    'sigma_syntax_validator',
+    'stix2_pattern_syntax_validator',
+    'sigma_queries',
+    'dbl_spamhaus',
+    'vulners',
+    'yara_query',
+    'macaddress_io',
+    'intel471',
+    'backscatter_io',
+    'btc_scam_check',
+    'hibp',
+    'greynoise',
+    'macvendors',
+    'qrcode',
+    'ocr_enrich',
+    'pdf_enrich',
+    'docx_enrich',
+    'xlsx_enrich',
+    'pptx_enrich',
+    'ods_enrich',
+    'odt_enrich',
+    'joesandbox_submit',
+    'joesandbox_query',
+    'urlhaus',
+    'virustotal_public',
+    'apiosintds',
+    'urlscan',
+    'securitytrails',
+    'apivoid',
+    'assemblyline_submit',
+    'assemblyline_query',
+    'ransomcoindb',
+    'malwarebazaar',
+    'lastline_query',
+    'lastline_submit',
+    'sophoslabs_intelix',
+    'cytomic_orion',
+    'censys_enrich',
+    'trustar_enrich',
+    'recordedfuture',
+    'html_to_markdown',
+    'socialscan',
+    'passive_ssh',
+    'qintel_qsentry',
+    'mwdb',
+    'hashlookup',
+    'mmdb_lookup',
+    'ipqs_fraud_and_risk_scoring',
+    'clamav',
+    'jinja_template_rendering',
+    'hyasinsight',
+    'variotdbs',
+    'crowdsec',
+    'extract_url_components',
+    'ipinfo',
+    'whoisfreaks',
+    'ip2locationio',
+    'stairwell',
+    'google_threat_intelligence',
+    'vulnerability_lookup',
+    'vysion',
+    'mcafee_insights_enrich',
+    'threatfox',
+    'yeti',
+    'abuseipdb',
+    'vmware_nsx',
+    'sigmf_expand',
+    'google_safe_browsing',
+    'google_search',
+    'whois',
+    'triage_submit',
+    'virustotal_upload',
+    'malshare_upload',
+    'convert_markdown_to_pdf',
+    'onion_lookup',
+]
 
 
 minimum_required_fields = ('type', 'uuid', 'value')

diff --git a/misp_modules/modules/expansion/onion_lookup.py b/misp_modules/modules/expansion/onion_lookup.py
@@ -14,22 +14,25 @@
         # 'url',
         # Any other Attribute type...
     ],
-    'format': 'misp_standard'
+    'format': 'misp_standard',
 }
 
 moduleinfo = {
     'version': '1',
+    'author': 'Sami Mokaddem',
+    'name': 'Onion Lookup',
     'author': 'MISP',
-    'description': 'MISP module using the MISP standard. Uses the onion-lookup service to get information about an onion',
+    'description': 'MISP module using the MISP standard. Uses the onion-lookup service to get information about an onion.',
     'module-type': [  # possible module-types: 'expansion', 'hover' or both
         'expansion',
-        'hover'
-    ]
+        'hover',
+    ],
+    'references': ['https://onion.ail-project.org/'],
+    'logo': 'onion.png'
 }
 
 # config fields that your code expects from the site admin
-moduleconfig = [
-]
+moduleconfig = []
 
 
 def getDetails(onion_address):
@@ -47,6 +50,8 @@ def getDetails(onion_address):
   ],
 }
 '''
+
+
 def createObject(onion_details):
     misp_object = MISPObject('tor-hiddenservice')
     misp_object.comment = 'custom-comment2'
@@ -62,7 +67,6 @@ def createObject(onion_details):
     return misp_object
 
 
-
 def enrichOnion(misp_event, attribute):
     onion_address = attribute['value']
     onion_details = getDetails(onion_address)
@@ -85,7 +89,9 @@ def handler(q=False):
 
     # Input sanity check
     if not request.get('attribute') or not check_input_attribute(request['attribute']):
-        return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'}
+        return {
+            'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'
+        }
     attribute = request['attribute']
 
     # Make sure the Attribute's type is one of the expected type
@@ -112,4 +118,3 @@ def introspection():
 def version():
     moduleinfo['config'] = moduleconfig
     return moduleinfo
-
diff --git a/misp_modules/modules/import_mod/lastline_import.py b/misp_modules/modules/import_mod/lastline_import.py
@@ -17,8 +17,9 @@
     "analysis_link": {
         "type": "String",
         "errorMessage": "Expected analysis link",
-        "message": "The link to a Lastline analysis"
-    },
+        "message": "The link to a Lastline analysis",
+        "required": True
+    }
 }
 
 inputSource = []

diff --git a/misp_modules/modules/import_mod/openiocimport.py b/misp_modules/modules/import_mod/openiocimport.py
@@ -4,14 +4,17 @@
 from pymisp.tools import openioc
 
 misperrors = {'error': 'Error'}
-userConfig = {'not save ioc': {'type': 'Boolean',
-                               'message': 'If you check this box, IOC file will not save as an attachment in MISP'
-                               },
-              'default tag': {
-                  'type': 'String',
-                  'message': 'Add tags spaced by a comma (tlp:white,misp:threat-level="no-risk")',
-                  'validation': '0'}
-              }
+userConfig = {
+    'not save ioc': {
+        'type': 'Boolean',
+        'message': 'If you check this box, IOC file will not save as an attachment in MISP'
+    },
+    'default tag': {
+        'type': 'String',
+        'message': 'Add tags spaced by a comma (tlp:white,misp:threat-level="no-risk")',
+        'validation': '0'
+    }
+}
 
 inputSource = ['file']
 

diff --git a/misp_modules/modules/import_mod/taxii21.py b/misp_modules/modules/import_mod/taxii21.py
@@ -4,13 +4,15 @@
 import collections
 import itertools
 import json
-import misp_modules.lib.stix2misp
-from pathlib import Path
 import re
-import stix2.v20
+import requests
 import taxii2client
 import taxii2client.exceptions
-import requests
+from pathlib import Path
+from misp_stix_converter import (
+    ExternalSTIX2toMISPParser, InternalSTIX2toMISPParser, _is_stix2_from_misp)
+from stix2.v20 import Bundle as Bundle_v20
+from stix2.v21 import Bundle as Bundle_v21
 
 
 class ConfigError(Exception):
@@ -24,13 +26,13 @@ class ConfigError(Exception):
 misperrors = {'error': 'Error'}
 
 moduleinfo = {
-    'version': '0.1',
+    'version': '0.2',
     'author': 'Abc',
     'description': 'Import content from a TAXII 2.1 server',
     'module-type': ['import'],
     'name': 'TAXII 2.1 Import',
     'logo': '',
-    'requirements': [],
+    'requirements': ['misp-lib-stix2', 'misp-stix'],
     'features': '',
     'references': [],
     'input': '',
@@ -40,14 +42,15 @@ class ConfigError(Exception):
 mispattributes = {
     'inputSource': [],
     'output': ['MISP objects'],
-    'format': 'misp_standard',
+    'format': 'misp_standard'
 }
 
 
 userConfig = {
     "url": {
         "type": "String",
         "message": "A TAXII 2.1 collection URL",
+        "required": True
     },
     "added_after": {
         "type": "String",
@@ -234,9 +237,9 @@ def _get_config(config):
     # STIX->MISP converter currently only supports STIX 2.0, so let's force
     # spec_version="2.0".
     if not spec_version:
-        spec_version = "2.0"
-    elif spec_version != "2.0":
-        raise ConfigError('Only spec_version="2.0" is supported for now.')
+        spec_version = "2.1"
+    if spec_version not in ("2.0", "2.1"):
+       raise ConfigError('Only spec versions "2.0" and "2.1" are valid versions.')
 
     if (username and not password) or (not username and password):
         raise ConfigError(
@@ -307,14 +310,16 @@ def _query_taxii(config):
     # memory usage.
     stix_objects = list(limited_stix_objects)
 
-    # The STIX 2.0 converter wants a 2.0 bundle.  (Hope the TAXII server isn't
-    # returning 2.1 objects!)
-    bundle20 = stix2.v20.Bundle(stix_objects, allow_custom=True)
+    bundle = (Bundle_v21 if config.spec_version == '2.1' else Bundle_v20)(
+        stix_objects, allow_custom=True
+    )
 
-    converter = misp_modules.lib.stix2misp.ExternalStixParser()
-    converter.handler(
-        bundle20, None, [0, "event", str(_synonymsToTagNames_path)]
+    converter = (
+        InternalSTIX2toMISPParser() if _is_stix2_from_misp(bundle.objects)
+        else ExternalSTIX2toMISPParser()
     )
+    converter.load_stix_bundle(bundle)
+    converter.parse_stix_bundle(single_event=True)
 
     attributes = [
         _pymisp_to_json_serializable(attr)