From 56eb72ddd7ffd912ab55e372716c24b2b17066ce Mon Sep 17 00:00:00 2001 From: Stefano Ortolani Date: Wed, 5 Mar 2025 16:30:23 +0000 Subject: [PATCH] feat: add new version API endpoint --- .gitchangelog.rc | 289 -------------------------- .github/workflows/release-package.yml | 14 +- .github/workflows/test-package.yml | 10 +- misp_modules/__init__.py | 18 +- misp_modules/__main__.py | 40 ++-- misp_modules/helpers/__init__.py | 0 misp_modules/helpers/cache.py | 99 --------- poetry.lock | 26 +-- pyproject.toml | 3 +- 9 files changed, 43 insertions(+), 456 deletions(-) delete mode 100644 .gitchangelog.rc delete mode 100644 misp_modules/helpers/__init__.py delete mode 100644 misp_modules/helpers/cache.py diff --git a/.gitchangelog.rc b/.gitchangelog.rc deleted file mode 100644 index 19d9b85bb..000000000 --- a/.gitchangelog.rc +++ /dev/null @@ -1,289 +0,0 @@ -# -*- coding: utf-8; mode: python -*- -## -## Format -## -## ACTION: [AUDIENCE:] COMMIT_MSG [!TAG ...] -## -## Description -## -## ACTION is one of 'chg', 'fix', 'new' -## -## Is WHAT the change is about. -## -## 'chg' is for refactor, small improvement, cosmetic changes... -## 'fix' is for bug fixes -## 'new' is for new features, big improvement -## -## AUDIENCE is optional and one of 'dev', 'usr', 'pkg', 'test', 'doc'|'docs' -## -## Is WHO is concerned by the change. -## -## 'dev' is for developpers (API changes, refactors...) -## 'usr' is for final users (UI changes) -## 'pkg' is for packagers (packaging changes) -## 'test' is for testers (test only related changes) -## 'doc' is for doc guys (doc only changes) -## -## COMMIT_MSG is ... well ... the commit message itself. -## -## TAGs are additionnal adjective as 'refactor' 'minor' 'cosmetic' -## -## They are preceded with a '!' or a '@' (prefer the former, as the -## latter is wrongly interpreted in github.) Commonly used tags are: -## -## 'refactor' is obviously for refactoring code only -## 'minor' is for a very meaningless change (a typo, adding a comment) -## 'cosmetic' is for cosmetic driven change (re-indentation, 80-col...) -## 'wip' is for partial functionality but complete subfunctionality. -## -## Example: -## -## new: usr: support of bazaar implemented -## chg: re-indentend some lines !cosmetic -## new: dev: updated code to be compatible with last version of killer lib. -## fix: pkg: updated year of licence coverage. -## new: test: added a bunch of test around user usability of feature X. -## fix: typo in spelling my name in comment. !minor -## -## Please note that multi-line commit message are supported, and only the -## first line will be considered as the "summary" of the commit message. So -## tags, and other rules only applies to the summary. The body of the commit -## message will be displayed in the changelog without reformatting. - - -## -## ``ignore_regexps`` is a line of regexps -## -## Any commit having its full commit message matching any regexp listed here -## will be ignored and won't be reported in the changelog. -## -ignore_regexps = [ - r'@minor', r'!minor', - r'@cosmetic', r'!cosmetic', - r'@refactor', r'!refactor', - r'@wip', r'!wip', - r'^([cC]hg|[fF]ix|[nN]ew)\s*:\s*[p|P]kg:', - r'^([cC]hg|[fF]ix|[nN]ew)\s*:\s*[d|D]ev:', - r'^(.{3,3}\s*:)?\s*[fF]irst commit.?\s*$', - ] - - -## ``section_regexps`` is a list of 2-tuples associating a string label and a -## list of regexp -## -## Commit messages will be classified in sections thanks to this. Section -## titles are the label, and a commit is classified under this section if any -## of the regexps associated is matching. -## -## Please note that ``section_regexps`` will only classify commits and won't -## make any changes to the contents. So you'll probably want to go check -## ``subject_process`` (or ``body_process``) to do some changes to the subject, -## whenever you are tweaking this variable. -## -section_regexps = [ - ('New', [ - r'^[nN]ew\s*:\s*((dev|use?r|pkg|test|doc|docs)\s*:\s*)?([^\n]*)$', - ]), - ('Changes', [ - r'^[cC]hg\s*:\s*((dev|use?r|pkg|test|doc|docs)\s*:\s*)?([^\n]*)$', - ]), - ('Fix', [ - r'^[fF]ix\s*:\s*((dev|use?r|pkg|test|doc|docs)\s*:\s*)?([^\n]*)$', - ]), - - ('Other', None ## Match all lines - ), - -] - - -## ``body_process`` is a callable -## -## This callable will be given the original body and result will -## be used in the changelog. -## -## Available constructs are: -## -## - any python callable that take one txt argument and return txt argument. -## -## - ReSub(pattern, replacement): will apply regexp substitution. -## -## - Indent(chars=" "): will indent the text with the prefix -## Please remember that template engines gets also to modify the text and -## will usually indent themselves the text if needed. -## -## - Wrap(regexp=r"\n\n"): re-wrap text in separate paragraph to fill 80-Columns -## -## - noop: do nothing -## -## - ucfirst: ensure the first letter is uppercase. -## (usually used in the ``subject_process`` pipeline) -## -## - final_dot: ensure text finishes with a dot -## (usually used in the ``subject_process`` pipeline) -## -## - strip: remove any spaces before or after the content of the string -## -## - SetIfEmpty(msg="No commit message."): will set the text to -## whatever given ``msg`` if the current text is empty. -## -## Additionally, you can `pipe` the provided filters, for instance: -#body_process = Wrap(regexp=r'\n(?=\w+\s*:)') | Indent(chars=" ") -#body_process = Wrap(regexp=r'\n(?=\w+\s*:)') -#body_process = noop -body_process = ReSub(r'((^|\n)[A-Z]\w+(-\w+)*: .*(\n\s+.*)*)+$', r'') | strip - - -## ``subject_process`` is a callable -## -## This callable will be given the original subject and result will -## be used in the changelog. -## -## Available constructs are those listed in ``body_process`` doc. -subject_process = (strip | - ReSub(r'^([cC]hg|[fF]ix|[nN]ew)\s*:\s*((dev|use?r|pkg|test|doc|docs)\s*:\s*)?([^\n@]*)(@[a-z]+\s+)*$', r'\4') | - SetIfEmpty("No commit message.") | ucfirst | final_dot) - - -## ``tag_filter_regexp`` is a regexp -## -## Tags that will be used for the changelog must match this regexp. -## -tag_filter_regexp = r'^v[0-9]+\.[0-9]+\.[0-9]+$' - - - -## ``unreleased_version_label`` is a string or a callable that outputs a string -## -## This label will be used as the changelog Title of the last set of changes -## between last valid tag and HEAD if any. -unreleased_version_label = "%%version%% (unreleased)" - - -## ``output_engine`` is a callable -## -## This will change the output format of the generated changelog file -## -## Available choices are: -## -## - rest_py -## -## Legacy pure python engine, outputs ReSTructured text. -## This is the default. -## -## - mustache() -## -## Template name could be any of the available templates in -## ``templates/mustache/*.tpl``. -## Requires python package ``pystache``. -## Examples: -## - mustache("markdown") -## - mustache("restructuredtext") -## -## - makotemplate() -## -## Template name could be any of the available templates in -## ``templates/mako/*.tpl``. -## Requires python package ``mako``. -## Examples: -## - makotemplate("restructuredtext") -## -#output_engine = rest_py -#output_engine = mustache("restructuredtext") -output_engine = mustache("markdown") -#output_engine = makotemplate("restructuredtext") - - -## ``include_merge`` is a boolean -## -## This option tells git-log whether to include merge commits in the log. -## The default is to include them. -include_merge = True - - -## ``log_encoding`` is a string identifier -## -## This option tells gitchangelog what encoding is outputed by ``git log``. -## The default is to be clever about it: it checks ``git config`` for -## ``i18n.logOutputEncoding``, and if not found will default to git's own -## default: ``utf-8``. -#log_encoding = 'utf-8' - - -## ``publish`` is a callable -## -## Sets what ``gitchangelog`` should do with the output generated by -## the output engine. ``publish`` is a callable taking one argument -## that is an interator on lines from the output engine. -## -## Some helper callable are provided: -## -## Available choices are: -## -## - stdout -## -## Outputs directly to standard output -## (This is the default) -## -## - FileInsertAtFirstRegexMatch(file, pattern, idx=lamda m: m.start()) -## -## Creates a callable that will parse given file for the given -## regex pattern and will insert the output in the file. -## ``idx`` is a callable that receive the matching object and -## must return a integer index point where to insert the -## the output in the file. Default is to return the position of -## the start of the matched string. -## -## - FileRegexSubst(file, pattern, replace, flags) -## -## Apply a replace inplace in the given file. Your regex pattern must -## take care of everything and might be more complex. Check the README -## for a complete copy-pastable example. -## -# publish = FileInsertIntoFirstRegexMatch( -# "CHANGELOG.rst", -# r'/(?P[0-9]+\.[0-9]+(\.[0-9]+)?)\s+\([0-9]+-[0-9]{2}-[0-9]{2}\)\n--+\n/', -# idx=lambda m: m.start(1) -# ) -#publish = stdout - - -## ``revs`` is a list of callable or a list of string -## -## callable will be called to resolve as strings and allow dynamical -## computation of these. The result will be used as revisions for -## gitchangelog (as if directly stated on the command line). This allows -## to filter exaclty which commits will be read by gitchangelog. -## -## To get a full documentation on the format of these strings, please -## refer to the ``git rev-list`` arguments. There are many examples. -## -## Using callables is especially useful, for instance, if you -## are using gitchangelog to generate incrementally your changelog. -## -## Some helpers are provided, you can use them:: -## -## - FileFirstRegexMatch(file, pattern): will return a callable that will -## return the first string match for the given pattern in the given file. -## If you use named sub-patterns in your regex pattern, it'll output only -## the string matching the regex pattern named "rev". -## -## - Caret(rev): will return the rev prefixed by a "^", which is a -## way to remove the given revision and all its ancestor. -## -## Please note that if you provide a rev-list on the command line, it'll -## replace this value (which will then be ignored). -## -## If empty, then ``gitchangelog`` will act as it had to generate a full -## changelog. -## -## The default is to use all commits to make the changelog. -#revs = ["^1.0.3", ] -#revs = [ -# Caret( -# FileFirstRegexMatch( -# "CHANGELOG.rst", -# r"(?P[0-9]+\.[0-9]+(\.[0-9]+)?)\s+\([0-9]+-[0-9]{2}-[0-9]{2}\)\n--+\n")), -# "HEAD" -#] -revs = [] diff --git a/.github/workflows/release-package.yml b/.github/workflows/release-package.yml index 90e23e602..2d7729d45 100644 --- a/.github/workflows/release-package.yml +++ b/.github/workflows/release-package.yml @@ -9,7 +9,7 @@ on: jobs: release: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout repository @@ -44,14 +44,14 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} docs: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout repository uses: actions/checkout@v4 - name: Install packages - run: sudo apt-get install libpoppler-cpp-dev libzbar0 tesseract-ocr yara + run: sudo apt-get install libpoppler-cpp-dev - name: Set up Python 3.12 uses: actions/setup-python@v5 @@ -73,7 +73,7 @@ jobs: path: site/ deploy-gh-pages: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: docs permissions: @@ -90,13 +90,13 @@ jobs: uses: actions/deploy-pages@v4 build: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout repository uses: actions/checkout@v4 - name: Install packages - run: sudo apt-get install libpoppler-cpp-dev libzbar0 tesseract-ocr yara + run: sudo apt-get install libgl1 libpoppler-cpp-dev libpoppler-cpp0v5 libzbar0 tesseract-ocr - name: Set up Python 3.12 uses: actions/setup-python@v5 @@ -119,7 +119,7 @@ jobs: path: dist/ publish-to-pypi: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: build permissions: diff --git a/.github/workflows/test-package.yml b/.github/workflows/test-package.yml index 92475df3c..8cfee41a7 100644 --- a/.github/workflows/test-package.yml +++ b/.github/workflows/test-package.yml @@ -11,14 +11,14 @@ on: jobs: docs: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout repository uses: actions/checkout@v4 - name: Install packages - run: sudo apt-get install libpoppler-cpp-dev libzbar0 tesseract-ocr yara + run: sudo apt-get install libpoppler-cpp-dev - name: Set up Python 3.12 uses: actions/setup-python@v5 @@ -32,7 +32,7 @@ jobs: run: make generate_docs test: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 strategy: fail-fast: false @@ -45,7 +45,7 @@ jobs: uses: actions/checkout@v4 - name: Install packages - run: sudo apt-get install libpoppler-cpp-dev libzbar0 tesseract-ocr yara + run: sudo apt-get install libgl1 libpoppler-cpp-dev libpoppler-cpp0v5 libzbar0 tesseract-ocr - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 @@ -89,7 +89,7 @@ jobs: path: dist/ publish-to-test-pypi: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: test permissions: diff --git a/misp_modules/__init__.py b/misp_modules/__init__.py index 580248ce3..c651300a5 100644 --- a/misp_modules/__init__.py +++ b/misp_modules/__init__.py @@ -19,6 +19,7 @@ import enum import importlib import importlib.abc +import importlib.metadata import importlib.resources import importlib.util import pathlib @@ -43,6 +44,14 @@ class ModuleType(enum.Enum): ACTION_MOD = "action_mod" +def get_version() -> str: + """Return the version.""" + try: + return importlib.metadata.version("misp-modules") + except importlib.metadata.PackageNotFoundError: + raise ValueError + + def is_valid_module(module: importlib.abc.Traversable) -> bool: """Whether the reference is a valid module file.""" if not module.is_file(): @@ -65,15 +74,6 @@ def is_valid_module_type(module_type: importlib.abc.Traversable) -> bool: return True -def iterate_helpers( - helpers_dir: typing.Union[importlib.abc.Traversable, pathlib.Path], -) -> typing.Generator[importlib.abc.Traversable, None, None]: - """Iterate helpers and return helper references.""" - for helper in helpers_dir.iterdir(): - if is_valid_module(helper): - yield helper - - def iterate_modules( modules_dir: typing.Union[importlib.abc.Traversable, pathlib.Path], ) -> typing.Generator[tuple[importlib.abc.Traversable, importlib.abc.Traversable], None, None]: diff --git a/misp_modules/__main__.py b/misp_modules/__main__.py index b41ef5fdb..3ce7f6f99 100644 --- a/misp_modules/__main__.py +++ b/misp_modules/__main__.py @@ -114,11 +114,22 @@ def init_logger(debug: bool = False) -> None: LOGGER.addHandler(handler) -class Healthcheck(tornado.web.RequestHandler): - """Healthcheck handler.""" +class VersionCheck(tornado.web.RequestHandler): + """VersionCheck handler.""" def get(self): - LOGGER.debug("MISP Healthcheck request") + LOGGER.debug("VersionCheck request") + try: + self.write(orjson.dumps({"version": misp_modules.get_version()})) + except ValueError: + self.send_error(500) + + +class HealthCheck(tornado.web.RequestHandler): + """HealthCheck handler.""" + + def get(self): + LOGGER.debug("Healthcheck request") self.write(b'{"status": true}') @@ -143,7 +154,7 @@ def _build_handlers_data(cls) -> bytes: ) def get(self): - LOGGER.debug("MISP ListModules request") + LOGGER.debug("ListModules request") if not self.CACHE: self.CACHE = self._build_handlers_data() self.write(self.CACHE) @@ -159,7 +170,7 @@ class QueryModule(tornado.web.RequestHandler): @tornado_concurrent.run_on_executor def run_request(self, module_name, json_payload, dict_payload): - LOGGER.debug("MISP QueryModule %s request %s", module_name, json_payload) + LOGGER.debug("QueryModule %s request %s", module_name, json_payload) try: response = MODULES_HANDLERS[module_name].dict_handler(request=dict_payload) except AttributeError: @@ -211,22 +222,6 @@ def main(): # Load libraries as root modules misp_modules.promote_lib_to_root() - # Load helpers - for helper in misp_modules.iterate_helpers( - importlib.resources.files(__package__).joinpath(misp_modules.HELPERS_DIR) - ): - helper_name = os.path.splitext(helper.name)[0] - absolute_helper_name = ".".join([__package__, misp_modules.HELPERS_DIR, helper_name]) - try: - imported_helper = importlib.import_module(absolute_helper_name) - if test_error := imported_helper.selftest(): - raise ImportError(test_error) - except ImportError as e: - LOGGER.warning("Helper %s failed: %s", helper_name, e) - continue - HELPERS_HANDLERS[helper_name] = imported_helper - LOGGER.info("Helper %s loaded", helper_name) - # Load modules for module_type, module in misp_modules.iterate_modules( importlib.resources.files(__package__).joinpath(misp_modules.MODULES_DIR) @@ -262,7 +257,8 @@ def main(): [ (r"/modules", ListModules), (r"/query", QueryModule), - (r"/healthcheck", Healthcheck), + (r"/healthcheck", HealthCheck), + (r"/version", VersionCheck), ] ), max_buffer_size=MAX_BUFFER_SIZE, diff --git a/misp_modules/helpers/__init__.py b/misp_modules/helpers/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/misp_modules/helpers/cache.py b/misp_modules/helpers/cache.py deleted file mode 100644 index 8be1b7faf..000000000 --- a/misp_modules/helpers/cache.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# MISP modules helper - cache -# -# Copyright (C) 2016 Alexandre Dulaunoy -# Copyright (C) 2016 CIRCL - Computer Incident Response Center Luxembourg -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import hashlib -import os -import typing - -import redis - -DEFAULT_HOST = "127.0.0.1" -DEFAULT_PORT = 6379 -DEFAULT_DATABASE = 0 -DEFAULT_TIMEOUT = 5 - - -def create_connection(timeout: int = DEFAULT_TIMEOUT, **kwargs) -> redis.Redis: - return redis.Redis( - host=os.getenv("REDIS_BACKEND", DEFAULT_HOST), - password=os.getenv("REDIS_PW"), - port=int(os.getenv("REDIS_PORT", DEFAULT_PORT)), - db=int(os.getenv("REDIS_DATABASE", DEFAULT_DATABASE)), - socket_timeout=timeout, - socket_connect_timeout=timeout, - **kwargs, - ) - - -def selftest(enable: bool = True) -> typing.Union[str, None]: - if not enable: - return None - try: - r = create_connection(timeout=3) - r.ping() - except Exception: - return "Redis not running or not installed. Helper will be disabled." - else: - return None - - -def get(modulename=None, query=None, value=None, debug=False): - if modulename is None or query is None: - return False - r = create_connection(decode_responses=True) - h = hashlib.sha1() - h.update(query.encode("UTF-8")) - hv = h.hexdigest() - key = "m:{}:{}".format(modulename, hv) - - if not r.exists(key): - if debug: - print("Key {} added in cache".format(key)) - r.setex(key, 86400, value) - else: - if debug: - print("Cache hit with Key {}".format(key)) - - return r.get(key) - - -def flush(): - r = create_connection(decode_responses=True) - return r.flushdb() - - -if __name__ == "__main__": - import sys - - if selftest() is not None: - sys.exit() - else: - print("Selftest ok") - v = get(modulename="testmodule", query="abcdef", value="barfoo", debug=True) - if v == "barfoo": - print("Cache ok") - v = get(modulename="testmodule", query="abcdef") - print(v) - v = get(modulename="testmodule") - if not v: - print("Failed ok") - if flush(): - print("Cache flushed ok") diff --git a/poetry.lock b/poetry.lock index b551b22cd..d0437e208 100644 --- a/poetry.lock +++ b/poetry.lock @@ -293,10 +293,10 @@ test = ["astroid (>=2,<4)", "pytest", "pytest-cov", "pytest-xdist"] name = "async-timeout" version = "5.0.1" description = "Timeout context manager for asyncio programs" -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] -markers = "python_version == \"3.9\" or python_version == \"3.11\" and python_full_version < \"3.11.3\" or python_version == \"3.10\" or platform_system == \"Linux\" and python_full_version < \"3.11.3\" and platform_machine == \"aarch64\"" +markers = "(python_version == \"3.9\" or python_version == \"3.10\" or platform_system == \"Linux\" and python_version <= \"3.10\" and platform_machine == \"aarch64\") and extra == \"all\" and python_version < \"3.11\"" files = [ {file = "async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c"}, {file = "async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3"}, @@ -5672,26 +5672,6 @@ files = [ {file = "red-black-tree-mod-1.22.tar.gz", hash = "sha256:38e3652903a2bf96379c27c2082ca0b7b905158662dd7ef0c97f4fd93a9aa908"}, ] -[[package]] -name = "redis" -version = "5.2.1" -description = "Python client for Redis database and key-value store" -optional = false -python-versions = ">=3.8" -groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\"" -files = [ - {file = "redis-5.2.1-py3-none-any.whl", hash = "sha256:ee7e1056b9aea0f04c6c2ed59452947f34c4940ee025f5dd83e6a6418b6989e4"}, - {file = "redis-5.2.1.tar.gz", hash = "sha256:16f2e22dff21d5125e8481515e386711a34cbec50f0e44413dd7d9c060a54e0f"}, -] - -[package.dependencies] -async-timeout = {version = ">=4.0.3", markers = "python_full_version < \"3.11.3\""} - -[package.extras] -hiredis = ["hiredis (>=3.0.0)"] -ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==23.2.1)", "requests (>=2.31.0)"] - [[package]] name = "referencing" version = "0.36.2" @@ -7497,4 +7477,4 @@ all = ["apiosintds", "assemblyline_client", "backscatter", "blockchain", "censys [metadata] lock-version = "2.1" python-versions = ">=3.9,<3.13" -content-hash = "b8a0449438694de58b9e1267e0f39f74f7ff36cbf38393e374ee0b890824a219" +content-hash = "7807c375091da6ca6575c270da83edec55c1b2d8bf4b865282dc448a9e97aaaf" diff --git a/pyproject.toml b/pyproject.toml index a836611df..b621f5eb8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "misp-modules" -version = "3.0.0" +version = "3.0.1" description = "MISP modules are autonomous modules that can be used for expansion and other services in MISP" authors = [ {name = "Alexandre Dulaunoy", email = "alexandre.dulaunoy@circl.lu"} @@ -20,7 +20,6 @@ dependencies = [ ## core dependencies "orjson", "psutil", - "redis", "tornado", ## minimum dependencies "beautifulsoup4",