From 03870b877ed50f8bf0246f89ff7846ecc5a2f9f6 Mon Sep 17 00:00:00 2001 From: Stefano Ortolani Date: Wed, 5 Mar 2025 16:30:23 +0000 Subject: [PATCH] new: add new version API endpoint --- .github/workflows/release-package.yml | 14 ++-- .github/workflows/test-package.yml | 10 +-- misp_modules/__init__.py | 18 ++--- misp_modules/__main__.py | 40 +++++------ misp_modules/helpers/__init__.py | 0 misp_modules/helpers/cache.py | 99 --------------------------- poetry.lock | 26 +------ pyproject.toml | 3 +- 8 files changed, 43 insertions(+), 167 deletions(-) delete mode 100644 misp_modules/helpers/__init__.py delete mode 100644 misp_modules/helpers/cache.py diff --git a/.github/workflows/release-package.yml b/.github/workflows/release-package.yml index 90e23e602..2d7729d45 100644 --- a/.github/workflows/release-package.yml +++ b/.github/workflows/release-package.yml @@ -9,7 +9,7 @@ on: jobs: release: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout repository @@ -44,14 +44,14 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} docs: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout repository uses: actions/checkout@v4 - name: Install packages - run: sudo apt-get install libpoppler-cpp-dev libzbar0 tesseract-ocr yara + run: sudo apt-get install libpoppler-cpp-dev - name: Set up Python 3.12 uses: actions/setup-python@v5 @@ -73,7 +73,7 @@ jobs: path: site/ deploy-gh-pages: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: docs permissions: @@ -90,13 +90,13 @@ jobs: uses: actions/deploy-pages@v4 build: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout repository uses: actions/checkout@v4 - name: Install packages - run: sudo apt-get install libpoppler-cpp-dev libzbar0 tesseract-ocr yara + run: sudo apt-get install libgl1 libpoppler-cpp-dev libpoppler-cpp0v5 libzbar0 tesseract-ocr - name: Set up Python 3.12 uses: actions/setup-python@v5 @@ -119,7 +119,7 @@ jobs: path: dist/ publish-to-pypi: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: build permissions: diff --git a/.github/workflows/test-package.yml b/.github/workflows/test-package.yml index 92475df3c..8cfee41a7 100644 --- a/.github/workflows/test-package.yml +++ b/.github/workflows/test-package.yml @@ -11,14 +11,14 @@ on: jobs: docs: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout repository uses: actions/checkout@v4 - name: Install packages - run: sudo apt-get install libpoppler-cpp-dev libzbar0 tesseract-ocr yara + run: sudo apt-get install libpoppler-cpp-dev - name: Set up Python 3.12 uses: actions/setup-python@v5 @@ -32,7 +32,7 @@ jobs: run: make generate_docs test: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 strategy: fail-fast: false @@ -45,7 +45,7 @@ jobs: uses: actions/checkout@v4 - name: Install packages - run: sudo apt-get install libpoppler-cpp-dev libzbar0 tesseract-ocr yara + run: sudo apt-get install libgl1 libpoppler-cpp-dev libpoppler-cpp0v5 libzbar0 tesseract-ocr - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 @@ -89,7 +89,7 @@ jobs: path: dist/ publish-to-test-pypi: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: test permissions: diff --git a/misp_modules/__init__.py b/misp_modules/__init__.py index 580248ce3..c651300a5 100644 --- a/misp_modules/__init__.py +++ b/misp_modules/__init__.py @@ -19,6 +19,7 @@ import enum import importlib import importlib.abc +import importlib.metadata import importlib.resources import importlib.util import pathlib @@ -43,6 +44,14 @@ class ModuleType(enum.Enum): ACTION_MOD = "action_mod" +def get_version() -> str: + """Return the version.""" + try: + return importlib.metadata.version("misp-modules") + except importlib.metadata.PackageNotFoundError: + raise ValueError + + def is_valid_module(module: importlib.abc.Traversable) -> bool: """Whether the reference is a valid module file.""" if not module.is_file(): @@ -65,15 +74,6 @@ def is_valid_module_type(module_type: importlib.abc.Traversable) -> bool: return True -def iterate_helpers( - helpers_dir: typing.Union[importlib.abc.Traversable, pathlib.Path], -) -> typing.Generator[importlib.abc.Traversable, None, None]: - """Iterate helpers and return helper references.""" - for helper in helpers_dir.iterdir(): - if is_valid_module(helper): - yield helper - - def iterate_modules( modules_dir: typing.Union[importlib.abc.Traversable, pathlib.Path], ) -> typing.Generator[tuple[importlib.abc.Traversable, importlib.abc.Traversable], None, None]: diff --git a/misp_modules/__main__.py b/misp_modules/__main__.py index b41ef5fdb..3ce7f6f99 100644 --- a/misp_modules/__main__.py +++ b/misp_modules/__main__.py @@ -114,11 +114,22 @@ def init_logger(debug: bool = False) -> None: LOGGER.addHandler(handler) -class Healthcheck(tornado.web.RequestHandler): - """Healthcheck handler.""" +class VersionCheck(tornado.web.RequestHandler): + """VersionCheck handler.""" def get(self): - LOGGER.debug("MISP Healthcheck request") + LOGGER.debug("VersionCheck request") + try: + self.write(orjson.dumps({"version": misp_modules.get_version()})) + except ValueError: + self.send_error(500) + + +class HealthCheck(tornado.web.RequestHandler): + """HealthCheck handler.""" + + def get(self): + LOGGER.debug("Healthcheck request") self.write(b'{"status": true}') @@ -143,7 +154,7 @@ def _build_handlers_data(cls) -> bytes: ) def get(self): - LOGGER.debug("MISP ListModules request") + LOGGER.debug("ListModules request") if not self.CACHE: self.CACHE = self._build_handlers_data() self.write(self.CACHE) @@ -159,7 +170,7 @@ class QueryModule(tornado.web.RequestHandler): @tornado_concurrent.run_on_executor def run_request(self, module_name, json_payload, dict_payload): - LOGGER.debug("MISP QueryModule %s request %s", module_name, json_payload) + LOGGER.debug("QueryModule %s request %s", module_name, json_payload) try: response = MODULES_HANDLERS[module_name].dict_handler(request=dict_payload) except AttributeError: @@ -211,22 +222,6 @@ def main(): # Load libraries as root modules misp_modules.promote_lib_to_root() - # Load helpers - for helper in misp_modules.iterate_helpers( - importlib.resources.files(__package__).joinpath(misp_modules.HELPERS_DIR) - ): - helper_name = os.path.splitext(helper.name)[0] - absolute_helper_name = ".".join([__package__, misp_modules.HELPERS_DIR, helper_name]) - try: - imported_helper = importlib.import_module(absolute_helper_name) - if test_error := imported_helper.selftest(): - raise ImportError(test_error) - except ImportError as e: - LOGGER.warning("Helper %s failed: %s", helper_name, e) - continue - HELPERS_HANDLERS[helper_name] = imported_helper - LOGGER.info("Helper %s loaded", helper_name) - # Load modules for module_type, module in misp_modules.iterate_modules( importlib.resources.files(__package__).joinpath(misp_modules.MODULES_DIR) @@ -262,7 +257,8 @@ def main(): [ (r"/modules", ListModules), (r"/query", QueryModule), - (r"/healthcheck", Healthcheck), + (r"/healthcheck", HealthCheck), + (r"/version", VersionCheck), ] ), max_buffer_size=MAX_BUFFER_SIZE, diff --git a/misp_modules/helpers/__init__.py b/misp_modules/helpers/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/misp_modules/helpers/cache.py b/misp_modules/helpers/cache.py deleted file mode 100644 index 8be1b7faf..000000000 --- a/misp_modules/helpers/cache.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# MISP modules helper - cache -# -# Copyright (C) 2016 Alexandre Dulaunoy -# Copyright (C) 2016 CIRCL - Computer Incident Response Center Luxembourg -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import hashlib -import os -import typing - -import redis - -DEFAULT_HOST = "127.0.0.1" -DEFAULT_PORT = 6379 -DEFAULT_DATABASE = 0 -DEFAULT_TIMEOUT = 5 - - -def create_connection(timeout: int = DEFAULT_TIMEOUT, **kwargs) -> redis.Redis: - return redis.Redis( - host=os.getenv("REDIS_BACKEND", DEFAULT_HOST), - password=os.getenv("REDIS_PW"), - port=int(os.getenv("REDIS_PORT", DEFAULT_PORT)), - db=int(os.getenv("REDIS_DATABASE", DEFAULT_DATABASE)), - socket_timeout=timeout, - socket_connect_timeout=timeout, - **kwargs, - ) - - -def selftest(enable: bool = True) -> typing.Union[str, None]: - if not enable: - return None - try: - r = create_connection(timeout=3) - r.ping() - except Exception: - return "Redis not running or not installed. Helper will be disabled." - else: - return None - - -def get(modulename=None, query=None, value=None, debug=False): - if modulename is None or query is None: - return False - r = create_connection(decode_responses=True) - h = hashlib.sha1() - h.update(query.encode("UTF-8")) - hv = h.hexdigest() - key = "m:{}:{}".format(modulename, hv) - - if not r.exists(key): - if debug: - print("Key {} added in cache".format(key)) - r.setex(key, 86400, value) - else: - if debug: - print("Cache hit with Key {}".format(key)) - - return r.get(key) - - -def flush(): - r = create_connection(decode_responses=True) - return r.flushdb() - - -if __name__ == "__main__": - import sys - - if selftest() is not None: - sys.exit() - else: - print("Selftest ok") - v = get(modulename="testmodule", query="abcdef", value="barfoo", debug=True) - if v == "barfoo": - print("Cache ok") - v = get(modulename="testmodule", query="abcdef") - print(v) - v = get(modulename="testmodule") - if not v: - print("Failed ok") - if flush(): - print("Cache flushed ok") diff --git a/poetry.lock b/poetry.lock index b551b22cd..d0437e208 100644 --- a/poetry.lock +++ b/poetry.lock @@ -293,10 +293,10 @@ test = ["astroid (>=2,<4)", "pytest", "pytest-cov", "pytest-xdist"] name = "async-timeout" version = "5.0.1" description = "Timeout context manager for asyncio programs" -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] -markers = "python_version == \"3.9\" or python_version == \"3.11\" and python_full_version < \"3.11.3\" or python_version == \"3.10\" or platform_system == \"Linux\" and python_full_version < \"3.11.3\" and platform_machine == \"aarch64\"" +markers = "(python_version == \"3.9\" or python_version == \"3.10\" or platform_system == \"Linux\" and python_version <= \"3.10\" and platform_machine == \"aarch64\") and extra == \"all\" and python_version < \"3.11\"" files = [ {file = "async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c"}, {file = "async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3"}, @@ -5672,26 +5672,6 @@ files = [ {file = "red-black-tree-mod-1.22.tar.gz", hash = "sha256:38e3652903a2bf96379c27c2082ca0b7b905158662dd7ef0c97f4fd93a9aa908"}, ] -[[package]] -name = "redis" -version = "5.2.1" -description = "Python client for Redis database and key-value store" -optional = false -python-versions = ">=3.8" -groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" -files = [ - {file = "redis-5.2.1-py3-none-any.whl", hash = "sha256:ee7e1056b9aea0f04c6c2ed59452947f34c4940ee025f5dd83e6a6418b6989e4"}, - {file = "redis-5.2.1.tar.gz", hash = "sha256:16f2e22dff21d5125e8481515e386711a34cbec50f0e44413dd7d9c060a54e0f"}, -] - -[package.dependencies] -async-timeout = {version = ">=4.0.3", markers = "python_full_version < \"3.11.3\""} - -[package.extras] -hiredis = ["hiredis (>=3.0.0)"] -ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==23.2.1)", "requests (>=2.31.0)"] - [[package]] name = "referencing" version = "0.36.2" @@ -7497,4 +7477,4 @@ all = ["apiosintds", "assemblyline_client", "backscatter", "blockchain", "censys [metadata] lock-version = "2.1" python-versions = ">=3.9,<3.13" -content-hash = "b8a0449438694de58b9e1267e0f39f74f7ff36cbf38393e374ee0b890824a219" +content-hash = "7807c375091da6ca6575c270da83edec55c1b2d8bf4b865282dc448a9e97aaaf" diff --git a/pyproject.toml b/pyproject.toml index a836611df..b621f5eb8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "misp-modules" -version = "3.0.0" +version = "3.0.1" description = "MISP modules are autonomous modules that can be used for expansion and other services in MISP" authors = [ {name = "Alexandre Dulaunoy", email = "alexandre.dulaunoy@circl.lu"} @@ -20,7 +20,6 @@ dependencies = [ ## core dependencies "orjson", "psutil", - "redis", "tornado", ## minimum dependencies "beautifulsoup4",