Improve compatibility and upgrade python to 3.12

Changes:
* Remove vysion (not compatible with python 3.12 and no public repository)
* Remove stiximport (requires archaic version of pymisp)
* Update Python to 3.12
* Pin Numpy to 1.X
* Add missing dependencies
* Commit lock file
* Update requirements file

.github/workflows/python-package.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
     steps:
     - name: Install packages
@@ -28,9 +28,13 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install flake8 pytest
+        pip install flake8 pytest
+        pip install pipenv
+        sed -i "s/python_version.*/python_version = \"${{ matrix.python-version }}\"/" Pipfile
+        pipenv lock
+        pipenv requirements > requirements.txt
         # pyfaul must be installed manually (?)
-        pip install -r REQUIREMENTS pyfaup
+        pip install -r requirements.txt pyfaup
         pip install .
     - name: Lint with flake8
       run: |

Pipfile

@@ -10,6 +10,10 @@ pytest = "*"
 flake8 = "*"
 
 [packages]
+numpy = "<2.0.0"
+matplotlib = "*"
+sigmf = "*"
+pysafebrowsing = "*"
 dnspython = "*"
 requests = { extras = ["security"], version = "*" }
 urlarchiver = "*"
@@ -79,4 +83,4 @@ openpyxl = "*"
 slack-sdk = "3.27.1"
 
 [requires]
-python_version = "3.7"
+python_version = "3.12"

README.md

@@ -100,7 +100,6 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj
 * [VMware NSX](misp_modules/modules/expansion/vmware_nsx.py) - a module to enrich a file or URL with VMware NSX Defender.
 * [VulnDB](misp_modules/modules/expansion/vulndb.py) - a module to query [VulnDB](https://www.riskbasedsecurity.com/).
 * [Vulners](misp_modules/modules/expansion/vulners.py) - an expansion module to expand information about CVEs using Vulners API.
-* [Vysion](misp_modules/modules/expansion/vysion.py) - an expansion module to add dark web intelligence using Vysion API.
 * [whois](misp_modules/modules/expansion/whois.py) - a module to query a local instance of [uwhois](https://github.com/rafiot/uwhoisd).
 * [whoisfreaks](misp_modules/modules/expansion/whoisfreaks.py) - An expansion module for [whoisfreaks](https://whoisfreaks.com/) that will provide an enriched analysis of the provided domain, including WHOIS and DNS information.
 * [wikidata](misp_modules/modules/expansion/wiki.py) - a [wikidata](https://www.wikidata.org) expansion module.

REQUIREMENTS

@@ -1,188 +1,181 @@
-aiohttp>=3.9.0
-aiosignal==1.3.1 ; python_version >= '3.7'
+-i https://pypi.org/simple
+aiohttp==3.9.5; python_version >= '3.8'
+aiosignal==1.3.1; python_version >= '3.7'
 antlr4-python3-runtime==4.9.3
-anyio==3.6.2 ; python_full_version >= '3.6.2'
-apiosintDS==2.0.3
+anyio==4.4.0; python_version >= '3.8'
+apiosintds==2.0.3; python_version >= '3.6'
 appdirs==1.4.4
-argcomplete==3.0.8 ; python_version >= '3.6'
-argparse==1.4.0
-assemblyline-client==4.5.1
-async-timeout==4.0.2 ; python_version >= '3.6'
-asynctest==0.13.0 ; python_version < '3.8'
-attrs==23.1.0 ; python_version >= '3.7'
-backoff==2.2.1 ; python_version >= '3.7' and python_version < '4.0'
-backports.zoneinfo==0.2.1 ; python_version < '3.9'
+assemblyline-client==4.9.3
+attrs==23.2.0; python_version >= '3.7'
+backoff==1.11.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 backscatter==0.2.4
-beautifulsoup4==4.12.2
-bidict==0.22.1 ; python_version >= '3.7'
+beautifulsoup4==4.12.3; python_full_version >= '3.6.0'
+bidict==0.23.1; python_version >= '3.8'
 blockchain==1.4.4
-censys==2.2.2
-certifi>=2023.7.22 ; python_version >= '3.6'
-cffi==1.15.1
-chardet==5.1.0
-charset-normalizer==3.1.0 ; python_full_version >= '3.7.0'
+cattrs==23.2.3; python_version >= '3.8'
+censys==2.0.9; python_full_version >= '3.6.2' and python_version < '4.0'
+certifi==2024.7.4; python_version >= '3.6'
+cffi==1.16.0; platform_python_implementation != 'PyPy'
+chardet==5.2.0; python_version >= '3.7'
+charset-normalizer==3.3.2; python_full_version >= '3.7.0'
 clamd==1.0.2
-click==8.1.3 ; python_version >= '3.7'
+click==8.1.7; python_version >= '3.7'
 click-plugins==1.1.1
-colorama==0.4.6 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'
-colorclass==2.2.2 ; python_version >= '2.6'
+colorama==0.4.6; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'
+colorclass==2.2.2; python_version >= '2.6'
+commonmark==0.9.1
 compressed-rtf==1.0.6
-configparser==5.3.0 ; python_version >= '3.7'
-crowdstrike-falconpy==1.2.15
-cryptography>=41.0.2 ; python_version >= '3.6'
-dateparser==1.1.8 ; python_version >= '3.7'
-decorator==5.1.1 ; python_version >= '3.5'
-deprecated==1.2.14 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+configparser==7.0.0; python_version >= '3.8'
+contourpy==1.2.1; python_version >= '3.9'
+crowdstrike-falconpy==0.9.0; python_version >= '3.6'
+cryptography==42.0.8; python_version >= '3.7'
+cycler==0.12.1; python_version >= '3.8'
+deprecated==1.2.14; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 dnsdb2==1.1.4
-dnspython==2.3.0
-domaintools-api==1.0.1
+dnspython==2.6.1; python_version >= '3.8'
+domaintools-api==2.0.0; python_version >= '3.6'
 easygui==0.98.3
 ebcdic==1.1.1
 enum-compat==0.0.3
-et-xmlfile==1.1.0 ; python_version >= '3.6'
-extract-msg==0.45.0
+et-xmlfile==1.1.0; python_version >= '3.6'
+extract-msg==0.48.7
 ezodf==0.3.2
-filelock==3.12.0 ; python_version >= '3.7'
-frozenlist==1.3.3 ; python_version >= '3.7'
-future==0.18.3 ; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
-geoip2==4.7.0
-h11==0.14.0 ; python_version >= '3.7'
-httpcore==0.17.1 ; python_version >= '3.7'
-httplib2==0.22.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-httpx==0.24.1 ; python_version >= '3.7'
-idna==3.4 ; python_version >= '3.5'
-imapclient==2.3.1
-importlib-metadata==4.13.0 ; python_version < '3.8'
-importlib-resources==5.12.0 ; python_version < '3.9'
+filelock==3.15.4; python_version >= '3.8'
+fonttools==4.53.1; python_version >= '3.8'
+frozenlist==1.4.1; python_version >= '3.8'
+future==1.0.0; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'
+geoip2==4.8.0; python_version >= '3.8'
+h11==0.14.0; python_version >= '3.7'
+httpcore==1.0.5; python_version >= '3.8'
+httplib2==0.22.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+httpx==0.27.0; python_version >= '3.8'
+idna==3.7; python_version >= '3.5'
 isodate==0.6.1
-itsdangerous==2.1.2 ; python_version >= '3.7'
-jaraco.classes==3.2.3 ; python_version >= '3.7'
-jbxapi==3.21.0
-jeepney==0.8.0 ; sys_platform == 'linux'
-jinja2==3.1.2
-json-log-formatter==0.5.2 ; python_version >= '2.7'
-jsonschema==4.19.0 ; python_version >= '3.7'
-keyring==23.13.1 ; python_version >= '3.7'
-lark-parser==0.12.0
-lief==0.13.2
-lxml==4.9.2
+jbxapi==3.23.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+jinja2==3.1.2; python_version >= '3.7'
+json-log-formatter==1.0; python_version >= '3.6'
+jsonschema==4.23.0; python_version >= '3.8'
+jsonschema-specifications==2023.12.1; python_version >= '3.8'
+kiwisolver==1.4.5; python_version >= '3.7'
+lark==1.1.9; python_version >= '3.6'
+lief==0.14.1
+lxml==5.2.2; python_version >= '3.6'
 maclookup==1.0.3
-markdown-it-py==2.2.0 ; python_version >= '3.7'
 markdownify==0.5.3
-markupsafe==2.1.2 ; python_version >= '3.7'
-matplotlib==3.7.2 ; python_version >= '3.8'
-matplotlib==3.5.3 ; python_version == '3.7'
-mattermostdriver==7.3.2
-maxminddb==2.3.0 ; python_version >= '3.7'
-mdurl==0.1.2 ; python_version >= '3.7'
-.
-more-itertools==9.1.0 ; python_version >= '3.7'
-msoffcrypto-tool==5.0.1 ; python_version >= '3' and platform_python_implementation != 'PyPy' or (platform_system != 'Windows' and platform_system != 'Darwin')
-multidict==6.0.4 ; python_version >= '3.7'
-mwdblib==4.4.0
+markupsafe==2.1.5; python_version >= '3.7'
+matplotlib==3.9.1; python_version >= '3.9'
+mattermostdriver==7.3.2; python_version >= '3.5'
+maxminddb==2.6.2; python_version >= '3.8'
+-e .
+more-itertools==10.3.0; python_version >= '3.8'
+msoffcrypto-tool==5.4.1; platform_python_implementation != 'PyPy' or (python_version >= '3' and platform_system != 'Windows' and platform_system != 'Darwin')
+multidict==6.0.5; python_version >= '3.7'
+mwdblib==3.4.1
 ndjson==0.3.1
 np==1.0.2
-numpy==1.21.6 ; python_version < '3.10' and platform_machine == 'aarch64'
+numpy==1.26.4; python_version >= '3.9'
 oauth2==1.9.0.post1
-git+https://github.com/cartertemm/ODTReader.git/@49d6938693f6faa3ff09998f86dba551ae3a996b#egg=odtreader
-olefile==0.46 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-oletools==0.60.1
-opencv-python==4.7.0.72
-openpyxl==3.1.2
-packaging==23.1 ; python_version >= '3.7'
-pandas==1.5.3
-pandas-ods-reader==0.1.4
+odtreader@ git+https://github.com/cartertemm/ODTReader.git/@49d6938693f6faa3ff09998f86dba551ae3a996b
+olefile==0.47; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+oletools==0.60.2
+opencv-python==4.10.0.84; python_version >= '3.6'
+openpyxl==3.1.5; python_version >= '3.8'
+packaging==24.1; python_version >= '3.8'
+pandas==1.3.5; python_full_version >= '3.7.1'
+pandas-ods-reader==0.1.2
 passivetotal==2.5.9
 pcodedmp==1.2.6
 pdftotext==2.2.2
-pillow>=10.2.0
-pkgutil-resolve-name==1.3.10 ; python_version < '3.9'
-progressbar2==4.2.0 ; python_full_version >= '3.7.0'
-psutil==5.9.5 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-publicsuffixlist==0.10.0.20230828 ; python_version >= '2.6'
-git+https://github.com/D4-project/BGP-Ranking.git/@68de39f6c5196f796055c1ac34504054d688aa59#egg=pybgpranking&subdirectory=client
-pycountry==22.3.5
-pycparser==2.21
-pycryptodome==3.19.1 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-pycryptodomex==3.19.1 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+pillow==10.4.0; python_version >= '3.8'
+platformdirs==4.2.2; python_version >= '3.8'
+progressbar2==4.4.2; python_version >= '3.8'
+psutil==6.0.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
+publicsuffixlist==1.0.1.20240702; python_version >= '3.5'
+pybgpranking@ git+https://github.com/D4-project/BGP-Ranking.git/@68de39f6c5196f796055c1ac34504054d688aa59#subdirectory=client
+pycountry==22.3.5; python_version >= '3.6' and python_version < '4'
+pycparser==2.22; python_version >= '3.8'
+pycryptodome==3.20.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+pycryptodomex==3.20.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 pydeep2==0.5.1
-git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails
-pyeupi==1.1
+pydnstrails@ git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a
+pyeupi==1.3.0; python_version >= '3.8' and python_version < '4.0'
 pyfaup==1.2
 pygeoip==0.3.2
-pygments==2.15.1 ; python_version >= '3.7'
-git+https://github.com/MISP/PyIntel471.git@917272fafa8e12102329faca52173e90c5256968#egg=pyintel471
-git+https://github.com/D4-project/IPASN-History.git/@a2853c39265cecdd0c0d16850bd34621c0551b87#egg=pyipasnhistory&subdirectory=client
-pymisp[email,fileobjects,openioc,pdfexport,url]==2.4.175
-git+https://github.com/sebdraven/pyonyphe@d1d6741f8ea4475f3bb77ff20c876f08839cabd1#egg=pyonyphe
-pyparsing==2.4.7 ; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
-pypdns==1.5.2
-pypssl==2.2
-pyrsistent==0.19.3 ; python_version >= '3.7'
-pysafebrowsing==0.1.2
-pytesseract==0.3.10
+pygments==2.18.0; python_version >= '3.8'
+pyintel471@ git+https://github.com/MISP/PyIntel471.git@917272fafa8e12102329faca52173e90c5256968
+pyipasnhistory@ git+https://github.com/D4-project/IPASN-History.git/@a2853c39265cecdd0c0d16850bd34621c0551b87#subdirectory=client
+pymisp[fileobjects,openioc,pdfexport,email,url]==2.4.194; python_version >= '3.8' and python_version < '4.0'
+pyonyphe@ git+https://github.com/sebdraven/pyonyphe@d1d6741f8ea4475f3bb77ff20c876f08839cabd1
+pyparsing==2.4.7; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'
+pypdns==2.2.3; python_version >= '3.8' and python_version < '4.0'
+pypssl==2.2; python_version >= '3.6' and python_version < '4.0'
+pysafebrowsing==0.1.3
+pytesseract==0.3.10; python_version >= '3.7'
 python-baseconv==1.2.2
-python-dateutil==2.8.2 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-python-docx==0.8.11
-python-engineio==4.4.1 ; python_version >= '3.6'
+python-dateutil==2.9.0.post0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'
+python-docx==1.1.2; python_version >= '3.7'
+python-engineio==4.9.1; python_version >= '3.6'
 python-magic==0.4.27
-python-pptx==0.6.21
-python-socketio[client]==5.8.0 ; python_version >= '3.6'
-python-utils==3.5.2 ; python_version >= '3.7'
-pytz==2023.3
-pytz-deprecation-shim==0.1.0.post0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
-pyyaml==6.0 ; python_version >= '3.6'
+python-pptx==0.6.23
+python-socketio[client]==5.11.3; python_version >= '3.8'
+python-utils==3.8.2; python_version >= '3.9'
+pytz==2024.1
+pyyaml==6.0.1; python_version >= '3.6'
 pyzbar==0.1.9
-pyzipper==0.3.6 ; python_version >= '3.5'
-rdflib==6.3.2 ; python_version >= '3.7' and python_version < '4.0'
+pyzipper==0.3.6; python_version >= '3.5'
+rdflib==7.0.0; python_full_version >= '3.8.1' and python_full_version < '4.0.0'
 red-black-tree-mod==1.20
-redis==4.5.5 ; python_version >= '3.7'
-regex==2023.5.5 ; python_version >= '3.6'
-reportlab==4.0.4
-requests[security]==2.31.0
-requests-cache==0.6.4 ; python_version >= '3.6'
-requests-file==1.5.1
-rich==13.3.5 ; python_full_version >= '3.7.0'
-rtfde==0.1.0
-secretstorage==3.3.3 ; sys_platform == 'linux'
-setuptools==67.7.2 ; python_version >= '3.7'
-shodan==1.29.1
-sigmatools==0.19.1
-sigmf==1.1.1
-simplejson==3.19.1 ; python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3'
-six==1.16.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-slack-sdk==3.27.1
-sniffio==1.3.0 ; python_version >= '3.7'
-socialscan==1.4
+redis==5.0.7; python_version >= '3.7'
+referencing==0.35.1; python_version >= '3.8'
+reportlab==4.2.2; python_version >= '3.7' and python_version < '4'
+requests[security]==2.32.3; python_version >= '3.8'
+requests-cache==1.2.1; python_version >= '3.8'
+requests-file==2.1.0
+rich==10.16.2; python_full_version >= '3.6.2' and python_full_version < '4.0.0'
+rpds-py==0.19.0; python_version >= '3.8'
+rtfde==0.1.2
+ruamel.yaml==0.18.6; python_version >= '3.7'
+ruamel.yaml.clib==0.2.8; platform_python_implementation == 'CPython' and python_version < '3.13'
+setuptools==70.3.0; python_version >= '3.8'
+shellingham==1.5.4; python_version >= '3.7'
+shodan==1.31.0
+sigmatools==0.23.1; python_version ~= '3.8'
+sigmf==1.2.2; python_version >= '3.7'
+simple-websocket==1.0.0; python_version >= '3.6'
+simplejson==3.19.2; python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2'
+six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'
+slack-sdk==3.27.1; python_version >= '3.6'
+sniffio==1.3.1; python_version >= '3.7'
+socialscan==1.4.0; python_version >= '3.6'
 socketio-client==0.5.7.4
-soupsieve==2.4.1 ; python_version >= '3.7'
-sparqlwrapper==2.0.0
-stix2==3.0.1
-stix2-patterns==2.0.0
-tabulate==0.9.0 ; python_version >= '3.7'
-tau-clients==0.3.0
+soupsieve==2.5; python_version >= '3.8'
+sparqlwrapper==2.0.0; python_version >= '3.7'
+stix2==3.0.1; python_version >= '3.6'
+stix2-patterns==2.0.0; python_version >= '3.6'
+tabulate==0.9.0; python_version >= '3.7'
+tau-clients==0.3.0; python_version >= '3.6'
 taxii2-client==2.3.0
-tldextract==3.4.3 ; python_version >= '3.7'
-tornado==6.2 ; python_version >= '3.7'
-tqdm==4.65.0 ; python_version >= '3.7'
-git+https://github.com/SteveClement/trustar-python.git@6954eae38e0c77eaeef26084b6c5fd033925c1c7#egg=trustar
-typing-extensions==4.5.0 ; python_version < '3.8'
-tzdata==2023.3 ; python_version >= '3.6'
-tzlocal==4.2 ; python_version >= '3.6'
+termcolor==2.4.0; python_version >= '3.8'
+tldextract==5.1.2; python_version >= '3.8'
+tornado==6.4.1; python_version >= '3.8'
+tqdm==4.66.4; python_version >= '3.7'
+trustar@ git+https://github.com/SteveClement/trustar-python.git@6954eae38e0c77eaeef26084b6c5fd033925c1c7
+typer==0.12.3; python_version >= '3.7'
+typing-extensions==4.12.2; python_version >= '3.8'
+tzlocal==5.2; python_version >= '3.8'
 unicodecsv==0.14.1
-url-normalize==1.4.3 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
+url-normalize==1.4.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
 urlarchiver==0.2
-urllib3==1.26.15 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
+urllib3==2.2.2; python_version >= '3.8'
 vt-graph-api==2.2.0
-vt-py==0.17.5
-vulners==2.0.10
-vysion==1.0.10
-wand==0.6.11
-websocket-client==1.5.1 ; python_version >= '3.7'
-websockets==11.0.3 ; python_version >= '3.7'
-wrapt==1.15.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-xlrd==2.0.1
-xlsxwriter==3.1.0 ; python_version >= '3.6'
+vt-py==0.18.2; python_full_version >= '3.7.0'
+vulners==2.1.7; python_version >= '3.8'
+wand==0.6.13
+websocket-client==1.8.0; python_version >= '3.8'
+websockets==12.0; python_version >= '3.8'
+wrapt==1.16.0; python_version >= '3.6'
+wsproto==1.2.0; python_full_version >= '3.7.0'
+xlrd==2.0.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
+xlsxwriter==3.2.0; python_version >= '3.6'
 yara-python==3.8.1
-yarl==1.9.2 ; python_version >= '3.7'
-zipp==3.15.0 ; python_version >= '3.7'
+yarl==1.9.4; python_version >= '3.7'