diff --git a/misp_modules/__init__.py b/misp_modules/__init__.py index 48ed3288..7fae3c26 100644 --- a/misp_modules/__init__.py +++ b/misp_modules/__init__.py @@ -17,22 +17,25 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import os.path import pathlib import signal import sys import importlib import logging +logging.captureWarnings(True) + import argparse import datetime import types import typing -import psutil import importlib.resources import importlib.resources.abc import importlib.util -import orjson as json +import orjson import pymisp +import psutil import tornado.web import tornado.process @@ -41,21 +44,27 @@ from concurrent.futures import ThreadPoolExecutor +# Constants +LIBRARY_DIR = "lib" MODULES_DIR = "modules" HELPERS_DIR = "helpers" +# See https://github.com/MISP/misp-modules/issues/662 +MAX_BUFFER_SIZE = 1073741824 + +# Global variables MODULES_HANDLERS = {} HELPERS_HANDLERS = {} +LOGGER = logging.getLogger("misp-modules") -log = logging.getLogger("misp-modules") - - -def handle_signal(sig, frame): +def handle_signal(sig: int, frame: types.FrameType) -> None: + """Handle the signal.""" _ = sig, frame ioloop.IOLoop.instance().add_callback_from_signal(ioloop.IOLoop.instance().stop) -def init_logger(debug=False): +def init_logger(debug: bool = False) -> None: + """Initialize the logger.""" formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") handler = logging.StreamHandler() handler.setFormatter(formatter) @@ -66,44 +75,71 @@ def init_logger(debug=False): access_log.setLevel(logging.INFO) access_log.addHandler(handler) + # Configure warning logs + warning_log = logging.getLogger("py.warnings") + warning_log.propagate = False + warning_log.setLevel(logging.ERROR) + warning_log.addHandler(handler) + # Set application log - log.addHandler(handler) - log.propagate = False - log.setLevel(logging.DEBUG if debug else logging.INFO) + LOGGER.propagate = False + LOGGER.setLevel(logging.DEBUG if debug else logging.INFO) + LOGGER.addHandler(handler) + + +def get_misp_modules_pid() -> int | None: + """Get the pid of any process that have `misp-modules` in the command line.""" + try: + for pid in psutil.pids(): + if any("misp-modules" in x for x in psutil.Process(pid).cmdline()): + return pid + return None + except psutil.AccessDenied: + return None + + +def is_valid_module(module: importlib.resources.abc.Traversable) -> bool: + """Whether the reference is a valid module file.""" + if not module.is_file(): + return False + if module.name == "__init__.py": + return False + if not module.name.endswith(".py"): + return False + return True + + +def is_valid_module_type(module_type: importlib.resources.abc.Traversable) -> bool: + """Whether the reference is a valid module type.""" + if not module_type.is_dir(): + return False + if module_type.name not in ("expansion", "export_mod", "import_mod", "action_mod"): + return False + return True def iterate_helpers( helpers_dir: importlib.resources.abc.Traversable | pathlib.Path, ) -> typing.Generator[importlib.resources.abc.Traversable, None, None]: + """Iterate helpers and return helper references.""" for helper in helpers_dir.iterdir(): - if not helper.is_file(): - continue - if helper.name == "__init__.py": - continue - if not helper.name.endswith(".py"): - continue - yield helper + if is_valid_module(helper): + yield helper def iterate_modules( modules_dir: importlib.resources.abc.Traversable | pathlib.Path, ) -> typing.Generator[tuple[importlib.resources.abc.Traversable, importlib.resources.abc.Traversable], None, None]: + """Iterate modules and return both module types and module references.""" for module_type in modules_dir.iterdir(): - if not module_type.is_dir(): - continue - if module_type.name == "__pycache__": - continue - for module in module_type.iterdir(): - if not module.is_file(): - continue - if module.name == "__init__.py": - continue - if not module.name.endswith(".py"): - continue - yield module_type, module + if is_valid_module_type(module_type): + for module in module_type.iterdir(): + if is_valid_module(module): + yield module_type, module def import_from_path(module_name: str, file_path: str) -> types.ModuleType: + """Import module from any point in the file system.""" spec = importlib.util.spec_from_file_location(module_name, file_path) module = importlib.util.module_from_spec(spec) sys.modules[module_name] = module @@ -112,75 +148,74 @@ def import_from_path(module_name: str, file_path: str) -> types.ModuleType: class Healthcheck(tornado.web.RequestHandler): + """Healthcheck handler.""" + def get(self): + LOGGER.debug("MISP Healthcheck request") self.write(b'{"status": true}') class ListModules(tornado.web.RequestHandler): + """ListModules handler.""" - _cached_json = None + CACHE = None + + @classmethod + def _build_handlers_data(cls) -> bytes: + return orjson.dumps([ + { + "name": module_name, + "type": MODULES_HANDLERS["type:" + module_name], + "mispattributes": MODULES_HANDLERS[module_name].introspection(), + "meta": MODULES_HANDLERS[module_name].version(), + } for module_name in MODULES_HANDLERS if not module_name.startswith("type:") + ]) def get(self): - if not self._cached_json: - ret = [] - for module_name in MODULES_HANDLERS: - if module_name.startswith("type:"): - continue - ret.append( - { - "name": module_name, - "type": MODULES_HANDLERS["type:" + module_name], - "mispattributes": MODULES_HANDLERS[module_name].introspection(), - "meta": MODULES_HANDLERS[module_name].version(), - } - ) - self._cached_json = json.dumps(ret) - - log.debug("MISP ListModules request") - self.write(self._cached_json) + LOGGER.debug("MISP ListModules request") + if not self.CACHE: + self.CACHE = self._build_handlers_data() + self.write(self.CACHE) class QueryModule(tornado.web.RequestHandler): + """QueryModule handler.""" + + DEFAULT_TIMEOUT = 300 - # Default value in Python 3.5 - # https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor - nb_threads = tornado.process.cpu_count() * 5 - executor = ThreadPoolExecutor(nb_threads) + # Never go above 32 + executor = ThreadPoolExecutor(max_workers=min(32, tornado.process.cpu_count() * 5)) @tornado_concurrent.run_on_executor def run_request(self, module_name, json_payload, dict_payload): - log.debug("MISP QueryModule %s request %s", module_name, json_payload) - module = MODULES_HANDLERS[module_name] - if getattr(module, "dict_handler", None): - # New method that avoids double JSON decoding, new modules should define dict_handler - response = module.dict_handler(request=dict_payload) - else: - response = module.handler(q=json_payload) - return json.dumps(response, default=pymisp.pymisp_json_default) + LOGGER.debug("MISP QueryModule %s request %s", module_name, json_payload) + try: + response = MODULES_HANDLERS[module_name].dict_handler(request=dict_payload) + except AttributeError: + response = MODULES_HANDLERS[module_name].handler(q=json_payload) + return orjson.dumps(response, default=pymisp.pymisp_json_default) @tornado.gen.coroutine def post(self): + json_payload = self.request.body + dict_payload = orjson.loads(json_payload) + timeout = datetime.timedelta(seconds=int(dict_payload.get("timeout", self.DEFAULT_TIMEOUT))) try: - json_payload = self.request.body - dict_payload = json.loads(json_payload) - if dict_payload.get("timeout"): - timeout = datetime.timedelta(seconds=int(dict_payload.get("timeout"))) - else: - timeout = datetime.timedelta(seconds=300) future = self.run_request(dict_payload["module"], json_payload, dict_payload) response = yield tornado.gen.with_timeout(timeout, future) self.write(response) except tornado.gen.TimeoutError: - log.warning("Timeout on {}".format(dict_payload["module"])) - self.write(json.dumps({"error": "Timeout."})) + LOGGER.warning("Timeout on {}".format(dict_payload["module"])) + self.write(orjson.dumps({"error": "Timeout."})) except Exception: - self.write(json.dumps({"error": "Something went wrong, look in the server logs for details"})) - log.exception("Something went wrong when processing query request") + self.write(orjson.dumps({"error": "Something went wrong, look in the server logs for details"})) + LOGGER.exception("Something went wrong when processing query request") finally: self.finish() def main(): + """Init function.""" global HELPERS_HANDLERS global MODULES_HANDLERS @@ -188,104 +223,93 @@ def main(): signal.signal(signal.SIGTERM, handle_signal) arg_parser = argparse.ArgumentParser(description="misp-modules", formatter_class=argparse.RawTextHelpFormatter) - arg_parser.add_argument("-t", "--test", default=False, action="store_true", help="Test mode") - arg_parser.add_argument("-d", "--debug", default=False, action="store_true", help="Enable debugging") - arg_parser.add_argument("-p", "--port", default=6666, help="port (default 6666)") + arg_parser.add_argument("-t", "--test", default=False, action="store_true", help="test mode") + arg_parser.add_argument("-d", "--debug", default=False, action="store_true", help="enable debugging") + arg_parser.add_argument("-p", "--port", type=int, default=6666, help="port (default 6666)") arg_parser.add_argument("-l", "--listen", default="localhost", help="address (default localhost)") - arg_parser.add_argument("-c", "--custom", default=None, help="custom root") + arg_parser.add_argument("-c", "--custom", default=None, help="custom modules root") args = arg_parser.parse_args() # Initialize init_logger(debug=args.debug) + # Load libraries as root modules + sys.path.append(str(importlib.resources.files(__package__).joinpath(LIBRARY_DIR))) + # Load helpers for helper in iterate_helpers(importlib.resources.files(__package__).joinpath(HELPERS_DIR)): - # remove extension from name - helper_name = helper.name.split(".")[0] + helper_name = os.path.splitext(helper.name)[0] absolute_helper_name = ".".join([__package__, HELPERS_DIR , helper_name]) try: imported_helper = importlib.import_module(absolute_helper_name) - test_error = imported_helper.selftest() - if test_error: + if test_error := imported_helper.selftest(): raise RuntimeError(test_error) except (ImportError, RuntimeError) as e: - log.warning("Helper %s failed: %s", helper_name, e) + LOGGER.warning("Helper %s failed: %s", helper_name, e) continue - else: - HELPERS_HANDLERS[helper_name] = imported_helper - log.info(f"Helper %s loaded", helper_name) + HELPERS_HANDLERS[helper_name] = imported_helper + LOGGER.info(f"Helper %s loaded", helper_name) # Load modules for module_type, module in iterate_modules(importlib.resources.files(__package__).joinpath(MODULES_DIR)): - # remove extension from name - module_name = module.name.split(".")[0] + module_name = os.path.splitext(module.name)[0] absolute_module_name = ".".join([__package__, MODULES_DIR , module_type.name, module_name]) try: imported_module = importlib.import_module(absolute_module_name) except ImportError as e: - log.warning("MISP module %s (type=%s) failed: %s", module_name, module_type.name, e) + LOGGER.warning("MISP module %s (type=%s) failed: %s", module_name, module_type.name, e) continue - else: - MODULES_HANDLERS[module_name] = imported_module - MODULES_HANDLERS[f"type:{module_name}"] = module_type.name - log.info("MISP module %s (type=%s) imported", module_name, module_type.name) + MODULES_HANDLERS[module_name] = imported_module + MODULES_HANDLERS[f"type:{module_name}"] = module_type.name + LOGGER.info("MISP module %s (type=%s) imported", module_name, module_type.name) # Load custom modules if args.custom: - log.info("Parsing custom modules from root directory: %s", args.custom) + LOGGER.info("Parsing custom modules from root directory: %s", args.custom) for module_type, module in iterate_modules(pathlib.Path(args.custom)): - # remove extension from name - module_name = module.name.split(".")[0] + module_name = os.path.splitext(module.name)[0] try: imported_module = import_from_path(module_name, str(module_type.joinpath(module.name))) except ImportError as e: - log.warning("CUSTOM MISP module %s (type=%s) failed: %s", module_name, module_type.name, e) + LOGGER.warning("CUSTOM MISP module %s (type=%s) failed: %s", module_name, module_type.name, e) continue - else: - MODULES_HANDLERS[module_name] = imported_module - MODULES_HANDLERS[f"type:{module_name}"] = module_type.name - log.info("CUSTOM MISP module %s (type=%s) imported", module_name, module_type.name) - - service = [ - (r"/modules", ListModules), - (r"/query", QueryModule), - (r"/healthcheck", Healthcheck), - ] - application = tornado.web.Application(service) + MODULES_HANDLERS[module_name] = imported_module + MODULES_HANDLERS[f"type:{module_name}"] = module_type.name + LOGGER.info("CUSTOM MISP module %s (type=%s) imported", module_name, module_type.name) + try: server = tornado.httpserver.HTTPServer( - application, max_buffer_size=1073741824 - ) # buffer size increase when large MISP event are submitted - GH issue 662 + tornado.web.Application([ + (r"/modules", ListModules), + (r"/query", QueryModule), + (r"/healthcheck", Healthcheck), + ]), + max_buffer_size=MAX_BUFFER_SIZE, + ) server.listen(args.port, args.listen) - except Exception as e: - if e.errno == 98: - pids = psutil.pids() - for pid in pids: - p = psutil.Process(pid) - if p.name() == "misp-modules": - print("\n\n\n") - print(e) - print("\nmisp-modules is still running as PID: {}\n".format(pid)) - print("Please kill accordingly:") - print("sudo kill {}".format(pid)) - return 1 - print(e) - print("misp-modules might still be running.") - else: - log.exception(f"Could not listen on {args.listen}:{args.port}") - return 1 - - log.info(f"MISP modules server started on {args.listen} port {args.port}") + except OSError as e: + match e.errno: + case 48 | 98: + LOGGER.exception("Could not listen on %s:%d", args.listen, args.port) + if pid := get_misp_modules_pid(): + LOGGER.exception("Dangling 'misp-modules' with pid %d found", pid) + case _: + LOGGER.exception("Unspecified OSError") + raise + except Exception: + LOGGER.exception("Unspecified Exception") + raise + + LOGGER.info(f"MISP modules server started on %s:%d", args.listen, args.port) if args.test: - log.info("MISP modules started in test-mode, quitting immediately.") + LOGGER.info("MISP modules started in test-mode, quitting immediately.") return 0 try: ioloop.IOLoop.instance().start() finally: ioloop.IOLoop.instance().stop() - - return 0 + return 0 if __name__ == "__main__": diff --git a/misp_modules/lib/__init__.py b/misp_modules/lib/__init__.py index 35ecac0e..e69de29b 100644 --- a/misp_modules/lib/__init__.py +++ b/misp_modules/lib/__init__.py @@ -1,12 +0,0 @@ -import joe_mapping -from .vt_graph_parser import * # noqa - -all = [ - "joe_parser", - "lastline_api", - "cof2misp", - "qintel_helper", - "dnstrails", - "onyphe", - "ODTReader", -] diff --git a/misp_modules/modules/__init__.py b/misp_modules/modules/__init__.py index 22a9989b..e69de29b 100644 --- a/misp_modules/modules/__init__.py +++ b/misp_modules/modules/__init__.py @@ -1,4 +0,0 @@ -#from .expansion import * # noqa -#from .import_mod import * # noqa -#from .export_mod import * # noqa -#from .action_mod import * # noqa diff --git a/misp_modules/modules/action_mod/__init__.py b/misp_modules/modules/action_mod/__init__.py index 53231a8b..e69de29b 100644 --- a/misp_modules/modules/action_mod/__init__.py +++ b/misp_modules/modules/action_mod/__init__.py @@ -1 +0,0 @@ -# __all__ = ["testaction", "mattermost", "slack"] diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index 36dd6aa5..ef6693cb 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -1,8 +1,3 @@ -import os -import sys - - - minimum_required_fields = ("type", "uuid", "value") checking_error = 'containing at least a "type" field and a "value" field' diff --git a/misp_modules/modules/expansion/cve.py b/misp_modules/modules/expansion/cve.py index e6e037c0..d8cc53ce 100755 --- a/misp_modules/modules/expansion/cve.py +++ b/misp_modules/modules/expansion/cve.py @@ -41,7 +41,7 @@ def handler(q=False): else: return {"error": "Vulnerability Lookup API not accessible."} parser = VulnerabilityLookupParser(attribute, api_url) - parser.parser_lookup_result(vulnerability) + parser.parse_lookup_result(vulnerability) return parser.get_results() diff --git a/misp_modules/modules/expansion/onyphe.py b/misp_modules/modules/expansion/onyphe.py index a60f7ad1..d63ed9fa 100644 --- a/misp_modules/modules/expansion/onyphe.py +++ b/misp_modules/modules/expansion/onyphe.py @@ -4,10 +4,7 @@ from pymisp import MISPEvent, MISPObject -try: - from onyphe import Onyphe -except ImportError: - print("pyonyphe module not installed.") +from misp_modules.lib.onyphe import Onyphe misperrors = {"error": "Error"} diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index bf19bc11..b4d9eaf2 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -2,10 +2,7 @@ import json -try: - from onyphe import Onyphe -except ImportError: - print("pyonyphe module not installed.") +from onyphe import Onyphe misperrors = {"error": "Error"} diff --git a/pyproject.toml b/pyproject.toml index e694cd69..36154c1f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -120,10 +120,9 @@ optional = true black = "*" codecov = "*" flake8 = "*" +ipdb = "*" nose = "*" pytest = "*" -ipdb = "*" -# decorator = "*" [tool.poetry.group.docs] optional = true