Pip to Poetry, Ubuntu Base image Bump, Dockerfile QA, Python 3.11 support (#2244)

* Docker base image update
* Docker file QA
* Github Actions version update
* Removed unwanted pinned repository
* Pip to Poetry migration
* Bump httptools
* Jump yara-python-dex
* Python 3.11 support

Author: ajinabraham

.dockerignore

@@ -9,6 +9,8 @@ tox.ini
 mobsf/db.sqlite3
 Dockerfile
 dist
+build
+.DS_Store
 *.egg-info
 docker-compose.yml
 *.md

.github/workflows/docker-latest.yml

@@ -11,23 +11,23 @@ jobs:
         steps:
         -
             name: Checkout
-            uses: actions/checkout@v2
+            uses: actions/checkout@v3
         - 
             name: Set up QEMU
-            uses: docker/setup-qemu-action@v1
+            uses: docker/setup-qemu-action@v2
         -   
             name: Set up Docker Buildx
-            uses: docker/setup-buildx-action@v1
+            uses: docker/setup-buildx-action@v2
         -
             name: Login to DockerHub
-            uses: docker/login-action@v1 
+            uses: docker/login-action@v2
             with:
                 username: ${{ secrets.DOCKER_USERNAME }}
                 password: ${{ secrets.DOCKER_TOKEN }}
         -
             name: Build and push
             id: docker_build
-            uses: docker/build-push-action@v2
+            uses: docker/build-push-action@v4
             with:
                 push: true
                 context: .

.github/workflows/docker-release.yml

@@ -10,16 +10,16 @@ jobs:
         steps:
         -
             name: Checkout
-            uses: actions/checkout@v2
+            uses: actions/checkout@v3
         - 
             name: Set up QEMU
-            uses: docker/setup-qemu-action@v1
+            uses: docker/setup-qemu-action@v2
         -   
             name: Set up Docker Buildx
-            uses: docker/setup-buildx-action@v1
+            uses: docker/setup-buildx-action@v2
         -
             name: Login to DockerHub
-            uses: docker/login-action@v1 
+            uses: docker/login-action@v2
             with:
                 username: ${{ secrets.DOCKER_USERNAME }}
                 password: ${{ secrets.DOCKER_TOKEN }}
@@ -36,7 +36,7 @@ jobs:
         -
             name: Build and push release
             id: docker_build
-            uses: docker/build-push-action@v2
+            uses: docker/build-push-action@v4
             with:
                 push: true
                 context: .

.github/workflows/mobsf-test.yml

@@ -11,52 +11,55 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04, macos-latest, windows-latest]
-        python-version: [3.9, '3.10']
-        # exclude:
-          # excludes py38, py39 on Windows
-          # - os: windows-latest
-          #   python-version: 3.8
+        os: [ubuntu-22.04, macos-latest, windows-latest]
+        python-version: [3.9, '3.10', '3.11']
 
     runs-on: ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v3
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Setup Pip
+
+    - name: Setup Pip and Poetry
       run: |
-        python -m pip install pip==22.3.1
+        python -m pip install pip==22.3.1 poetry==1.6.1
+
     - name: Lint
       if: startsWith(matrix.os, 'ubuntu')
       run: |
         python -m pip install --upgrade tox
         tox -e lint
+
     - name: Install Ubuntu Dependencies
       if: startsWith(matrix.os, 'ubuntu')
       run: |
-        sudo apt update && sudo apt install -y xfonts-75dpi xfonts-base
-        export WKHTML_URL=https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/
-        export WKHTML_DEB=wkhtmltox_0.12.6-1.focal_amd64.deb
+        sudo apt update && sudo apt install -y xfonts-75dpi xfonts-base libssl3
+        export WKHTML_URL=https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6.1-2/
+        export WKHTML_DEB=wkhtmltox_0.12.6.1-2.jammy_amd64.deb
         wget ${WKHTML_URL}${WKHTML_DEB} && sudo dpkg -i ${WKHTML_DEB} && rm -rf ${WKHTML_DEB}
+
     - name: Install macOS Dependencies
       if: startsWith(matrix.os, 'macOS')
       run: |
         brew install --cask wkhtmltopdf
+
     - name: Install Windows Dependencies
       if: startsWith(matrix.os, 'windows')
       run: |
         choco install wkhtmltopdf
+
     - name: Install Python dependencies
       run: |
-        pip install --use-deprecated=legacy-resolver -r requirements.txt
+        poetry install --only main --no-interaction --no-ansi || poetry install --only main --no-interaction --no-ansi || poetry install --only main --no-interaction --no-ansi
+
     - name: Migrate Database
       run: |
-        python manage.py makemigrations
-        python manage.py makemigrations StaticAnalyzer
-        python manage.py migrate
+        poetry run python manage.py makemigrations
+        poetry run python manage.py makemigrations StaticAnalyzer
+        poetry run python manage.py migrate
     - name: Unit Tests
       run: |
         git submodule update --init --recursive
-        python manage.py test mobsf
+        poetry run python manage.py test mobsf

.github/workflows/python-publish.yml

@@ -10,9 +10,9 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v3
       with:
         python-version: '3.x'
     - name: Install dependencies

Dockerfile

@@ -1,5 +1,5 @@
 # Base image
-FROM ubuntu:20.04
+FROM ubuntu:22.04
 
 # Labels and Credits
 LABEL \
@@ -10,16 +10,7 @@ LABEL \
     contributor_2="Vincent Nadal <[email protected]>" \
     description="Mobile Security Framework (MobSF) is an automated, all-in-one mobile application (Android/iOS/Windows) pen-testing, malware analysis and security assessment framework capable of performing static and dynamic analysis."
 
-# Environment vars
-ENV DEBIAN_FRONTEND="noninteractive" \
-    MOBSF_PLATFORM="docker" \
-    JDK_FILE="openjdk-16.0.1_linux-x64_bin.tar.gz" \
-    JDK_FILE_ARM="openjdk-16.0.1_linux-aarch64_bin.tar.gz" \
-    WKH_FILE="wkhtmltox_0.12.6-1.focal_amd64.deb" \
-    WKH_FILE_ARM="wkhtmltox_0.12.6-1.focal_arm64.deb" \
-    JAVA_HOME="/jdk-16.0.1"
-
-ENV PATH="$JAVA_HOME/bin:$PATH"
+ENV DEBIAN_FRONTEND=noninteractive
 
 # See https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#run
 RUN apt update -y && apt install -y  --no-install-recommends \
@@ -34,33 +25,46 @@ RUN apt update -y && apt install -y  --no-install-recommends \
     fontconfig \
     xfonts-75dpi \
     xfonts-base \
-    python3.9 \
+    python3 \
     python3-dev \
     python3-pip \
     wget \
     curl \
     git \
     jq \
-    android-tools-adb
-
-# Set locales
-RUN locale-gen en_US.UTF-8
-ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8'
+    android-tools-adb && \
+    locale-gen en_US.UTF-8 
+
+ENV MOBSF_USER=mobsf \
+    MOBSF_PLATFORM=docker \
+    MOBSF_ADB_BINARY=/usr/bin/adb \
+    JDK_FILE=openjdk-20.0.2_linux-x64_bin.tar.gz \
+    JDK_FILE_ARM=openjdk-20.0.2_linux-aarch64_bin.tar.gz \
+    WKH_FILE=wkhtmltox_0.12.6.1-2.jammy_amd64.deb \
+    WKH_FILE_ARM=wkhtmltox_0.12.6.1-2.jammy_arm64.deb \
+    JAVA_HOME=/jdk-20.0.2 \
+    PATH=$JAVA_HOME/bin:$PATH \
+    LANG=en_US.UTF-8 \
+    LANGUAGE=en_US:en \
+    LC_ALL=en_US.UTF-8 \
+    PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONFAULTHANDLER=1 \
+    POETRY_VERSION=1.6.1
 
 # Install wkhtmltopdf & OpenJDK
 ARG TARGETPLATFORM
 
 COPY scripts/install_java_wkhtmltopdf.sh .
 RUN ./install_java_wkhtmltopdf.sh
 
-RUN groupadd -g 9901 mobsf
-RUN adduser mobsf --shell /bin/false -u 9901 --ingroup mobsf --gecos "" --disabled-password
-
+RUN groupadd -g 9901 $MOBSF_USER
+RUN adduser $MOBSF_USER --shell /bin/false -u 9901 --ingroup $MOBSF_USER --gecos "" --disabled-password
 
-# Install Requirements
-COPY requirements.txt .
-RUN pip3 install --upgrade --no-cache-dir setuptools pip && \
-    pip3 install --quiet --no-cache-dir -r requirements.txt
+COPY poetry.lock pyproject.toml ./
+RUN python3 -m pip install --upgrade --no-cache-dir pip poetry==${POETRY_VERSION} && \
+    poetry config virtualenvs.create false && \
+    poetry install --only main --no-interaction --no-ansi
 
 # Cleanup
 RUN \
@@ -80,27 +84,23 @@ WORKDIR /home/mobsf/Mobile-Security-Framework-MobSF
 # Copy source code
 COPY . .
 
-# Set adb binary path and create apktool framework directory
-ENV MOBSF_ADB_BINARY=/usr/bin/adb
-RUN mkdir -p /home/mobsf/.local/share/apktool/framework
-
-# Postgres support is set to false by default
+# Check if Postgres support needs to be enabled.
+# Disabled by default
 ARG POSTGRES=False
+ENV POSTGRES_USER=postgres \
+    POSTGRES_PASSWORD=password \
+    POSTGRES_DB=mobsf \
+    POSTGRES_HOST=postgres
 
-ENV POSTGRES_USER=postgres
-ENV POSTGRES_PASSWORD=password
-ENV POSTGRES_DB=mobsf
-ENV POSTGRES_HOST=postgres
-
-# Check if Postgres support needs to be enabled
 RUN ./scripts/postgres_support.sh $POSTGRES
 
 HEALTHCHECK CMD curl --fail http://host.docker.internal:8000/ || exit 1
 
 # Expose MobSF Port and Proxy Port
 EXPOSE 8000 8000 1337 1337
 
-RUN chown -R mobsf:mobsf /home/mobsf/Mobile-Security-Framework-MobSF
+RUN chown -R $MOBSF_USER:$MOBSF_USER /home/mobsf
 USER mobsf
+
 # Run MobSF
 CMD ["/home/mobsf/Mobile-Security-Framework-MobSF/scripts/entrypoint.sh"]

README.md

@@ -7,7 +7,7 @@ Mobile Security Framework (MobSF) is an automated, all-in-one mobile application
 
 Made with ![Love](https://cloud.githubusercontent.com/assets/4301109/16754758/82e3a63c-4813-11e6-9430-6015d98aeaab.png) in India
 
-[![python](https://img.shields.io/badge/python-3.8+-blue.svg?logo=python&labelColor=yellow)](https://www.python.org/downloads/)
+[![python](https://img.shields.io/badge/python-3.9+-blue.svg?logo=python&labelColor=yellow)](https://www.python.org/downloads/)
 [![PyPI version](https://badge.fury.io/py/mobsf.svg)](https://badge.fury.io/py/mobsf)
 [![platform](https://img.shields.io/badge/platform-osx%2Flinux%2Fwindows-green.svg)](https://github.com/MobSF/Mobile-Security-Framework-MobSF/)
 [![License](https://img.shields.io/:license-GPL--3.0--only-blue.svg)](https://www.gnu.org/licenses/gpl-3.0.html)
@@ -25,7 +25,7 @@ Made with ![Love](https://cloud.githubusercontent.com/assets/4301109/16754758/82
 [![Blackhat Arsenal Asia 2015](https://img.shields.io/badge/Black%20Hat%20Arsenal-Asia%202015-blue.svg)](https://www.blackhat.com/asia-15/arsenal.html#yso-mobile-security-framework)
 [![Blackhat Arsenal Asia 2018](https://img.shields.io/badge/Black%20Hat%20Arsenal-Asia%202018-blue.svg)](https://www.blackhat.com/asia-18/arsenal.html#mobile-security-framework-mobsf)
 
-MobSF is also bundled with [Android Tamer](https://androidtamer.com/tamer4-release), [BlackArch](https://blackarch.org/mobile.html) and [Pentoo](https://www.pentoo.ch/).
+MobSF is also bundled with [Android Tamer](https://tamerplatform.com), [BlackArch](https://blackarch.org/mobile.html) and [Pentoo](https://www.pentoo.ch/).
 
 ## Support MobSF
 

mobsf/MobSF/init.py

@@ -10,7 +10,7 @@
 
 logger = logging.getLogger(__name__)
 
-VERSION = '3.7.5'
+VERSION = '3.7.6'
 BANNER = """
   __  __       _    ____  _____       _____ _____ 
  |  \/  | ___ | |__/ ___||  ___|_   _|___ /|___  |

mobsf/MobSF/settings.py

@@ -89,6 +89,7 @@
     'application/x-itunes-ipa',
     'application/x-zip-compressed',
     'application/x-ar',
+    'text/vnd.a',
     'binary/octet-stream',
 ]
 ZIP_MIME = [