CI: Update CI to include ubuntu 22.04; drop deprecated macos 10.15 (#9)

brendan-ward · Jan 7, 2023 · 94e3d99 · 94e3d99
1 parent 7b08d69
commit 94e3d99
Show file tree

Hide file tree

Showing 9 changed files with 213 additions and 49 deletions.
diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml
@@ -30,13 +30,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu-20.04", "macos-10.15"]
+        os: ["ubuntu-22.04", "ubuntu-20.04", "macos-12"]
         platform: ["x86_64"]
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Checkout submodules
         run: |
@@ -57,7 +57,7 @@ jobs:
               vendor/zip-archive
 
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -98,7 +98,7 @@ jobs:
         run: ccache --clear --set-config cache_dir=~/.ccache
 
       - name: Cache ccache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         env:
           cache-name: ccache-v1
         with:
@@ -114,7 +114,28 @@ jobs:
           ccache --max-size=2G --set-config cache_dir=~/.ccache
           ccache --show-stats --set-config cache_dir=~/.ccache
 
-      - name: Build package
+      - name: Set platform tag for Ubuntu 22.04
+        if: matrix.os == 'ubuntu-22.04'
+        run: echo "PLATFORM_TAG=ubuntu22_04_x86_64" >> $GITHUB_ENV
+
+      - name: Set platform tag for Ubuntu 20.04
+        if: matrix.os == 'ubuntu-20.04'
+        run: echo "PLATFORM_TAG=ubuntu20_04_x86_64">> $GITHUB_ENV
+
+      - name: Build package for Ubuntu
+        if: runner.os == 'Linux'
+        env:
+          CMAKE_CXX_COMPILER_LAUNCHER: ccache
+        run: |
+          sudo mkdir -p $BUILD_TEMP_DIR
+          sudo chown -R $(id -u):$(id -g) $BUILD_TEMP_DIR
+          pip install wheel
+          python setup.py bdist_wheel --plat-name=$PLATFORM_TAG
+          echo "List wheels"
+          ls ./dist
+
+      - name: Build package for MacOS
+        if: runner.os == 'macOS'
         env:
           CMAKE_CXX_COMPILER_LAUNCHER: ccache
         run: |
@@ -123,7 +144,7 @@ jobs:
           pip install wheel
           python setup.py bdist_wheel
 
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         with:
           path: ./dist/*.whl
 
@@ -134,19 +155,36 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu-20.04", "macos-10.15", "macos-11"]
-        python-version: ["3.8", "3.9", "3.10"]
+        os: ["ubuntu-22.04", "ubuntu-20.04", "macos-12", "macos-11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
+    env:
+      PYTHON_VERSION: ${{ matrix.python-version }}
 
     steps:
-      - name: Checkout
-        uses: actions/checkout@v2
+      - name: Install Ubuntu 22.04 runtime dependencies
+        if: matrix.os == 'ubuntu-22.04'
+        run: |
+          sudo apt-get update
+          sudo apt-get -y install \
+            libicu70 \
+            libcurl4 \
+            libjpeg-turbo8 \
+            libpng16-16 \
+            libprotobuf23 \
+            libuv1 \
+            libx11-6 \
+            libegl1 \
+            libopengl0 \
+            xvfb
+          echo "PLATFORM_TAG=ubuntu22_04_x86_64" >> $GITHUB_ENV
 
-      - name: Install Linux runtime dependencies
-        if: runner.os == 'Linux'
+      - name: Install Ubuntu 20.04 runtime dependencies
+        if: matrix.os == 'ubuntu-20.04'
         run: |
           sudo apt-get update
           sudo apt-get -y install \
             libicu66 \
+            libcurl4 \
             libjpeg-turbo8 \
             libpng16-16 \
             libprotobuf17 \
@@ -155,37 +193,43 @@ jobs:
             libegl1 \
             libopengl0 \
             xvfb
+          echo "PLATFORM_TAG=ubuntu20_04_x86_64" >> $GITHUB_ENV
 
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 
       - name: Download wheels from artifacts
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           path: wheelhouse
 
-      - name: Install dependencies and pymgl wheel
+      - name: Install dependencies
         shell: bash
         run: |
+          echo "PYTHON_SITE_PACKAGES=$pythonLocation/lib/python$PYTHON_VERSION/site-packages" >> $GITHUB_ENV
           python -m pip install pytest Pillow numpy pixelmatch python-dotenv
-          python -m pip install --find-links wheelhouse/artifact pymgl
-          python -m pip list
 
-      - name: Run Linux tests
+      - name: Install and test wheels on Ubuntu
         if: runner.os == 'Linux'
         run: |
-          cd ..
+          python -m pip install --find-links wheelhouse/artifact --platform $PLATFORM_TAG --target $PYTHON_SITE_PACKAGES --no-deps --upgrade pymgl
+          python -m pip list
+          python -m pip list --path $PYTHON_SITE_PACKAGES
           xvfb-run -a --server-args="-screen 0 1024x768x24 -ac +render -noreset" \
             python -m pytest --pyargs pymgl -v
 
-      - name: Run MacOS tests
+      - name: Install and test wheels on MacOS
         if: runner.os == 'macOS'
         run: |
+          python -m pip install --find-links wheelhouse/artifact pymgl
+          python -m pip list
           cd ..
           python -m pytest --pyargs pymgl -v
 
+  # NOTE: publish is limited to MacOS wheels because Linux wheels are not accepted
+  # for only Ubuntu, must be manylinux
   publish:
     name: Publish to GitHub / PyPI
     needs: [test-wheels]
@@ -194,7 +238,7 @@ jobs:
     if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/')
 
     steps:
-      - uses: actions/download-artifact@v2
+      - uses: actions/download-artifact@v3
         with:
           name: artifact
           path: wheels

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -17,13 +17,17 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu-20.04", "macos-11", "macos-10.15"]
+        os:
+          - "ubuntu-22.04"
+          - "ubuntu-20.04"
+          - "macos-12"
+          - "macos-11"
         platform: ["x86_64"]
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Checkout submodules
         run: |
@@ -44,7 +48,7 @@ jobs:
               vendor/zip-archive
 
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -85,7 +89,7 @@ jobs:
         run: ccache --clear --set-config cache_dir=~/.ccache
 
       - name: Cache ccache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         env:
           cache-name: ccache-v1
         with:

diff --git a/README.md b/README.md
@@ -36,24 +36,44 @@ python -m pip install pytest Pillow numpy pixelmatch python-dotenv
 python -m pytest --pyargs pymgl -v
 ```
 
-#### Ubuntu 20.04
+#### Ubuntu 22.04 & 20.04
 
 Due to the complexity of building manylinux wheels that include OpenGL and
-successfully compile `maplibre-gl-native`, wheels are only available for Ubuntu 20.04.
+successfully compile `maplibre-gl-native`, wheels are only available for
+Ubuntu 22.04 and 20.04.
 
 Wheels are available on the release page in Github. Download and install from there.
 
 Something like:
 
 ```bash
-pip install https://github.com/brendan-ward/pymgl/releases/download/<release>/pymgl-<release>-<Python version>-<Python version>-linux_x86_64.whl
+pip install https://github.com/brendan-ward/pymgl/releases/download/<release>/pymgl-<release>-<Python version>-<Python version>-ubuntu<Ubuntu version>_x86_64.whl
 ```
 
 You also need to install the following runtime dependencies:
 
+**Ubuntu 22.04:**
+
+```bash
+apt-get install
+    libicu70 \
+    libcurl4 \
+    libjpeg-turbo8 \
+    libpng16-16 \
+    libprotobuf23 \
+    libuv1 \
+    libx11-6 \
+    libegl1 \
+    libopengl0 \
+    xvfb
+```
+
+**Ubuntu 20.04:**
+
 ```bash
 apt-get install
     libicu66 \
+    libcurl4 \
     libjpeg-turbo8 \
     libpng16-16 \
     libprotobuf17 \
@@ -313,7 +333,7 @@ via `homebrew`:
 -   cmake
 -   ninja
 
-#### Developing on Linux (Ubuntu 20.04) requires the following binary libraries:
+#### Developing on Ubuntu 22.04 or 20.04 requires the following binary libraries:
 
 -   cmake
 -   ninja-build
@@ -330,7 +350,61 @@ via `homebrew`:
 
 To run on Linux, XVFB must also be running; otherwise the process will segfault.
 
-See `docker/Dockerfile` for more information.
+See [`docker/README.md`](./docker/README.md) for more information.
+
+#### Developing on other Linux versions
+
+See `.github/workflows/build_wheels.yml` for the overall structure for building
+and testing on Linux. You will most likely need to adapt the names of the
+packages to the package manager for your version of Linux.
+
+##### Debian Bullseye
+
+We've heard (#7) that it is possible to successfully build and operate PyMGL
+on Debian Bullseye with the following dependencies:
+
+Build:
+
+```bash
+apt-get -y install \
+    curl \
+    build-essential \
+    cmake \
+    ccache \
+    ninja-build \
+    pkg-config \
+    libcurl4-openssl-dev \
+    libicu-dev \
+    libturbojpeg0-dev \
+    libpng-dev \
+    libprotobuf-dev \
+    libuv1-dev \
+    libx11-dev \
+    libegl-dev \
+    libopengl-dev \
+    xvfb \
+    libjpeg-dev \
+    libsqlite3-dev \
+    libopengl0 \
+    git
+```
+
+Runtime:
+
+```bash
+apt install --no-install-recommends -y \
+    xvfb \
+    xauth \
+    curl \
+    libicu67 \
+    libjpeg-turbo-progs \
+    libpng16-16 \
+    libprotobuf23 \
+    libuv1 \
+    libx11-6 \
+    libegl1 \
+    libopengl0
+```
 
 ### PyBind11
 

diff --git a/docker/Dockerfile.dev → docker/Dockerfile.ubuntu20.04 b/docker/Dockerfile.dev → docker/Dockerfile.ubuntu20.04
@@ -3,6 +3,9 @@ FROM ubuntu:20.04
 WORKDIR /app
 
 ENV DISPLAY :99
+ENV DEBIAN_FRONTEND noninteractive
+ENV PYTHONMALLOC malloc
+ENV BUILD_TEMP_DIR /tmp/build
 
 RUN apt-get update && \
     apt-get -y install software-properties-common && \
@@ -15,8 +18,10 @@ RUN apt-get update && \
     ccache \
     ninja-build \
     pkg-config \
-    python3.9 \
-    python3.9-dev \
+    # install python 3.9
+    python3 \
+    python3-dev \
+    python3-pip \
     # required dependencies
     libcurl4-openssl-dev \
     libicu-dev \
@@ -31,13 +36,7 @@ RUN apt-get update && \
     xvfb \
     # debugging utilities
     valgrind && \
-    # install pip for python3.9
-    curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
-    python3.9 get-pip.py && \
     /usr/sbin/update-ccache-symlinks
 
 #  Install python test deps
-RUN pip3.9 install pytest pytest-benchmark pytest-valgrind python-dotenv Pillow numpy pixelmatch requests
-
-ENV BUILD_TEMP_DIR /tmp/build
-ENV PYTHONMALLOC malloc
+RUN python3 -m pip install pytest pytest-benchmark pytest-valgrind python-dotenv Pillow numpy pixelmatch requests