fix: [modules] many modules not loaded as python module

Author: cvandeplas

documentation/README.md

@@ -1944,23 +1944,24 @@ An expansion hover module to expand information about CVE id using Vulners API.
 
 -----
 
-#### [Vysion](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/vysion.py)
+#### [vysion](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/vysion.py)
 
 <img src=logos/vysion.png height=60>
 
 Module to enrich the information by making use of the Vysion API.
 - **features**:
->This module gets correlated information from our dark web intelligence database. With this you will get several objects containing information related to, for example, an organization victim of a ransomware attack.
->MISP objects containing title, link to our webapp and TOR, i2p or clearnet URLs.
+>This module gets correlated information from Byron Labs' dark web intelligence database. With this you will get several objects containing information related to, for example, an organization victim of a ransomware attack.
 - **input**:
->MISP Attribute which include: company(target-org), country, info, BTC, XMR and DASH address.
+>company(target-org), country, info, BTC, XMR and DASH address.
 - **output**:
 >MISP objects containing title, link to our webapp and TOR, i2p or clearnet URLs.
 - **references**:
->https://vysion.ai/
+> - https://vysion.ai/
+> - https://developers.vysion.ai/
+> - https://github.com/ByronLabs/vysion-cti/tree/main
 - **requirements**:
-> Vysion python library
-> Vysion API Key
+> - Vysion python library
+> - Vysion API Key
 
 -----
 

documentation/mkdocs/expansion.md

@@ -1941,6 +1941,27 @@ An expansion hover module to expand information about CVE id using Vulners API.
 
 -----
 
+#### [vysion](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/vysion.py)
+
+<img src=../logos/vysion.png height=60>
+
+Module to enrich the information by making use of the Vysion API.
+- **features**:
+>This module gets correlated information from Byron Labs' dark web intelligence database. With this you will get several objects containing information related to, for example, an organization victim of a ransomware attack.
+- **input**:
+>company(target-org), country, info, BTC, XMR and DASH address.
+- **output**:
+>MISP objects containing title, link to our webapp and TOR, i2p or clearnet URLs.
+- **references**:
+> - https://vysion.ai/
+> - https://developers.vysion.ai/
+> - https://github.com/ByronLabs/vysion-cti/tree/main
+- **requirements**:
+> - Vysion python library
+> - Vysion API Key
+
+-----
+
 #### [whois](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/whois.py)
 
 Module to query a local instance of uwhois (https://github.com/rafiot/uwhoisd).

misp_modules/modules/expansion/__init__.py

@@ -17,11 +17,13 @@
            'virustotal_public', 'apiosintds', 'urlscan', 'securitytrails', 'apivoid',
            'assemblyline_submit', 'assemblyline_query', 'ransomcoindb', 'malwarebazaar',
            'lastline_query', 'lastline_submit', 'sophoslabs_intelix', 'cytomic_orion', 'censys_enrich',
-           'trustar_enrich', 'recordedfuture', 'html_to_markdown', 'socialscan', 'passive-ssh',
+           'trustar_enrich', 'recordedfuture', 'html_to_markdown', 'socialscan', 'passive_ssh',
            'qintel_qsentry', 'mwdb', 'hashlookup', 'mmdb_lookup', 'ipqs_fraud_and_risk_scoring',
-           'clamav', 'jinja_template_rendering','hyasinsight', 'variotdbs', 'crowdsec',
+           'clamav', 'jinja_template_rendering', 'hyasinsight', 'variotdbs', 'crowdsec',
            'extract_url_components', 'ipinfo', 'whoisfreaks', 'ip2locationio', 'stairwell',
-           'google_threat_intelligence', 'vulnerability_lookup', 'vysion']
+           'google_threat_intelligence', 'vulnerability_lookup', 'vysion', 'mcafee_insights_enrich',
+           'threatfox', 'yeti', 'abuseipdb', 'vmware_nsx', 'sigmf_expand', 'google_safe_browsing',
+           'google_search']
 
 
 minimum_required_fields = ('type', 'uuid', 'value')

misp_modules/modules/expansion/passive-ssh.py renamed to misp_modules/modules/expansion/passive_ssh.py

@@ -16,5 +16,7 @@
     'cof2misp',
     'joe_import',
     'taxii21',
-    'url_import'
+    'url_import',
+    'vmray_summary_json_import',
+    'import_blueprint'
 ]

misp_modules/modules/expansion/sigmf-expand.py renamed to misp_modules/modules/expansion/sigmf_expand.py

@@ -0,0 +1,30 @@
+import os
+import unittest
+import requests
+
+
+class TestActions(unittest.TestCase):
+    """Unittest module for action modules"""
+    def setUp(self):
+        self.headers = {'Content-Type': 'application/json'}
+        self.url = "http://127.0.0.1:6666/"
+
+    def test_introspection(self):
+        """checks if all action modules are offered through the misp-modules service"""
+        try:
+            response = requests.get(self.url + "modules")
+            modules = [module["name"] for module in response.json()]
+            # list modules in the export_mod folder
+            export_mod_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'misp_modules', 'modules', "action_mod")
+            module_files = [file[:-3] for file in os.listdir(export_mod_path) if file.endswith(".py") if file not in ['__init__.py']]
+            missing = []
+            for module in module_files:
+                if module not in modules:
+                    missing.append(module)
+            self.assertEqual(missing, [], f"Missing modules in __init__: {missing}")
+        finally:
+            response.connection.close()
+
+
+if __name__ == "__main__":
+    unittest.main()

misp_modules/modules/import_mod/__init__.py

@@ -88,6 +88,22 @@ def get_values(response):
                 return values[0] if isinstance(values, list) else values
         return data['results'][0]['values']
 
+    def test_introspection(self):
+        """checks if all expansion modules are offered through the misp-modules service"""
+        try:
+            response = requests.get(self.url + "modules")
+            modules = [module["name"] for module in response.json()]
+            # list modules in the export_mod folder
+            export_mod_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'misp_modules', 'modules', "expansion")
+            module_files = [file[:-3] for file in os.listdir(export_mod_path) if file.endswith(".py") if file not in ['__init__.py']]
+            missing = []
+            for module in module_files:
+                if module not in modules:
+                    missing.append(module)
+            self.assertEqual(missing, [], f"Missing modules in __init__: {missing}")
+        finally:
+            response.connection.close()
+
     def test_apiosintds(self):
         self.skipTest("apiosintds is probably broken")
 

tests/test_actions.py

@@ -1,4 +1,3 @@
-"""Test module for the ThreatConnect Export module"""
 import base64
 import csv
 import io
@@ -35,8 +34,11 @@ def test_introspection(self):
             # list modules in the export_mod folder
             export_mod_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'misp_modules', 'modules', "export_mod")
             module_files = [file[:-3] for file in os.listdir(export_mod_path) if file.endswith(".py") if file not in ['__init__.py', 'testexport.py']]
+            missing = []
             for module in module_files:
-                self.assertIn(module, modules)
+                if module not in modules:
+                    missing.append(module)
+            self.assertEqual(missing, [], f"Missing modules in __init__: {missing}")
         finally:
             response.connection.close()
 

tests/test_expansions.py

@@ -0,0 +1,30 @@
+import os
+import unittest
+import requests
+
+
+class TestImports(unittest.TestCase):
+    """Unittest module for import modules"""
+    def setUp(self):
+        self.headers = {'Content-Type': 'application/json'}
+        self.url = "http://127.0.0.1:6666/"
+
+    def test_introspection(self):
+        """checks if all import modules are offered through the misp-modules service"""
+        try:
+            response = requests.get(self.url + "modules")
+            modules = [module["name"] for module in response.json()]
+            # list modules in the export_mod folder
+            export_mod_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'misp_modules', 'modules', "import_mod")
+            module_files = [file[:-3] for file in os.listdir(export_mod_path) if file.endswith(".py") if file not in ['__init__.py', 'testimport.py']]
+            missing = []
+            for module in module_files:
+                if module not in modules:
+                    missing.append(module)
+            self.assertEqual(missing, [], f"Missing modules in __init__: {missing}")
+        finally:
+            response.connection.close()
+
+
+if __name__ == "__main__":
+    unittest.main()