From 94e3d995c961e0fb96894ce807b0a8cfe985c2ed Mon Sep 17 00:00:00 2001 From: Brendan Ward Date: Fri, 6 Jan 2023 19:24:38 -0800 Subject: [PATCH] CI: Update CI to include ubuntu 22.04; drop deprecated macos 10.15 (#9) --- .github/workflows/build_wheels.yml | 88 ++++++++++++++----- .github/workflows/test.yml | 14 +-- README.md | 84 ++++++++++++++++-- ...{Dockerfile.dev => Dockerfile.ubuntu20.04} | 17 ++-- docker/Dockerfile.ubuntu22.04 | 43 +++++++++ docker/README.md | 10 +-- .../fixtures/example-style-bad-source.json | 2 +- pymgl/tests/test_style.py | 2 +- tests/StyleTest.cpp | 2 +- 9 files changed, 213 insertions(+), 49 deletions(-) rename docker/{Dockerfile.dev => Dockerfile.ubuntu20.04} (70%) create mode 100644 docker/Dockerfile.ubuntu22.04 diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 97fa43c..82ac379 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -30,13 +30,13 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu-20.04", "macos-10.15"] + os: ["ubuntu-22.04", "ubuntu-20.04", "macos-12"] platform: ["x86_64"] - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Checkout submodules run: | @@ -57,7 +57,7 @@ jobs: vendor/zip-archive - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} @@ -98,7 +98,7 @@ jobs: run: ccache --clear --set-config cache_dir=~/.ccache - name: Cache ccache - uses: actions/cache@v2 + uses: actions/cache@v3 env: cache-name: ccache-v1 with: @@ -114,7 +114,28 @@ jobs: ccache --max-size=2G --set-config cache_dir=~/.ccache ccache --show-stats --set-config cache_dir=~/.ccache - - name: Build package + - name: Set platform tag for Ubuntu 22.04 + if: matrix.os == 'ubuntu-22.04' + run: echo "PLATFORM_TAG=ubuntu22_04_x86_64" >> $GITHUB_ENV + + - name: Set platform tag for Ubuntu 20.04 + if: matrix.os == 'ubuntu-20.04' + run: echo "PLATFORM_TAG=ubuntu20_04_x86_64">> $GITHUB_ENV + + - name: Build package for Ubuntu + if: runner.os == 'Linux' + env: + CMAKE_CXX_COMPILER_LAUNCHER: ccache + run: | + sudo mkdir -p $BUILD_TEMP_DIR + sudo chown -R $(id -u):$(id -g) $BUILD_TEMP_DIR + pip install wheel + python setup.py bdist_wheel --plat-name=$PLATFORM_TAG + echo "List wheels" + ls ./dist + + - name: Build package for MacOS + if: runner.os == 'macOS' env: CMAKE_CXX_COMPILER_LAUNCHER: ccache run: | @@ -123,7 +144,7 @@ jobs: pip install wheel python setup.py bdist_wheel - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 with: path: ./dist/*.whl @@ -134,19 +155,36 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu-20.04", "macos-10.15", "macos-11"] - python-version: ["3.8", "3.9", "3.10"] + os: ["ubuntu-22.04", "ubuntu-20.04", "macos-12", "macos-11"] + python-version: ["3.8", "3.9", "3.10", "3.11"] + env: + PYTHON_VERSION: ${{ matrix.python-version }} steps: - - name: Checkout - uses: actions/checkout@v2 + - name: Install Ubuntu 22.04 runtime dependencies + if: matrix.os == 'ubuntu-22.04' + run: | + sudo apt-get update + sudo apt-get -y install \ + libicu70 \ + libcurl4 \ + libjpeg-turbo8 \ + libpng16-16 \ + libprotobuf23 \ + libuv1 \ + libx11-6 \ + libegl1 \ + libopengl0 \ + xvfb + echo "PLATFORM_TAG=ubuntu22_04_x86_64" >> $GITHUB_ENV - - name: Install Linux runtime dependencies - if: runner.os == 'Linux' + - name: Install Ubuntu 20.04 runtime dependencies + if: matrix.os == 'ubuntu-20.04' run: | sudo apt-get update sudo apt-get -y install \ libicu66 \ + libcurl4 \ libjpeg-turbo8 \ libpng16-16 \ libprotobuf17 \ @@ -155,37 +193,43 @@ jobs: libegl1 \ libopengl0 \ xvfb + echo "PLATFORM_TAG=ubuntu20_04_x86_64" >> $GITHUB_ENV - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Download wheels from artifacts - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: path: wheelhouse - - name: Install dependencies and pymgl wheel + - name: Install dependencies shell: bash run: | + echo "PYTHON_SITE_PACKAGES=$pythonLocation/lib/python$PYTHON_VERSION/site-packages" >> $GITHUB_ENV python -m pip install pytest Pillow numpy pixelmatch python-dotenv - python -m pip install --find-links wheelhouse/artifact pymgl - python -m pip list - - name: Run Linux tests + - name: Install and test wheels on Ubuntu if: runner.os == 'Linux' run: | - cd .. + python -m pip install --find-links wheelhouse/artifact --platform $PLATFORM_TAG --target $PYTHON_SITE_PACKAGES --no-deps --upgrade pymgl + python -m pip list + python -m pip list --path $PYTHON_SITE_PACKAGES xvfb-run -a --server-args="-screen 0 1024x768x24 -ac +render -noreset" \ python -m pytest --pyargs pymgl -v - - name: Run MacOS tests + - name: Install and test wheels on MacOS if: runner.os == 'macOS' run: | + python -m pip install --find-links wheelhouse/artifact pymgl + python -m pip list cd .. python -m pytest --pyargs pymgl -v + # NOTE: publish is limited to MacOS wheels because Linux wheels are not accepted + # for only Ubuntu, must be manylinux publish: name: Publish to GitHub / PyPI needs: [test-wheels] @@ -194,7 +238,7 @@ jobs: if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/') steps: - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v3 with: name: artifact path: wheels diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 50a452a..5545f52 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,13 +17,17 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu-20.04", "macos-11", "macos-10.15"] + os: + - "ubuntu-22.04" + - "ubuntu-20.04" + - "macos-12" + - "macos-11" platform: ["x86_64"] - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Checkout submodules run: | @@ -44,7 +48,7 @@ jobs: vendor/zip-archive - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} @@ -85,7 +89,7 @@ jobs: run: ccache --clear --set-config cache_dir=~/.ccache - name: Cache ccache - uses: actions/cache@v2 + uses: actions/cache@v3 env: cache-name: ccache-v1 with: diff --git a/README.md b/README.md index f5f826a..09b476b 100644 --- a/README.md +++ b/README.md @@ -36,24 +36,44 @@ python -m pip install pytest Pillow numpy pixelmatch python-dotenv python -m pytest --pyargs pymgl -v ``` -#### Ubuntu 20.04 +#### Ubuntu 22.04 & 20.04 Due to the complexity of building manylinux wheels that include OpenGL and -successfully compile `maplibre-gl-native`, wheels are only available for Ubuntu 20.04. +successfully compile `maplibre-gl-native`, wheels are only available for +Ubuntu 22.04 and 20.04. Wheels are available on the release page in Github. Download and install from there. Something like: ```bash -pip install https://github.com/brendan-ward/pymgl/releases/download//pymgl----linux_x86_64.whl +pip install https://github.com/brendan-ward/pymgl/releases/download//pymgl----ubuntu_x86_64.whl ``` You also need to install the following runtime dependencies: +**Ubuntu 22.04:** + +```bash +apt-get install + libicu70 \ + libcurl4 \ + libjpeg-turbo8 \ + libpng16-16 \ + libprotobuf23 \ + libuv1 \ + libx11-6 \ + libegl1 \ + libopengl0 \ + xvfb +``` + +**Ubuntu 20.04:** + ```bash apt-get install libicu66 \ + libcurl4 \ libjpeg-turbo8 \ libpng16-16 \ libprotobuf17 \ @@ -313,7 +333,7 @@ via `homebrew`: - cmake - ninja -#### Developing on Linux (Ubuntu 20.04) requires the following binary libraries: +#### Developing on Ubuntu 22.04 or 20.04 requires the following binary libraries: - cmake - ninja-build @@ -330,7 +350,61 @@ via `homebrew`: To run on Linux, XVFB must also be running; otherwise the process will segfault. -See `docker/Dockerfile` for more information. +See [`docker/README.md`](./docker/README.md) for more information. + +#### Developing on other Linux versions + +See `.github/workflows/build_wheels.yml` for the overall structure for building +and testing on Linux. You will most likely need to adapt the names of the +packages to the package manager for your version of Linux. + +##### Debian Bullseye + +We've heard (#7) that it is possible to successfully build and operate PyMGL +on Debian Bullseye with the following dependencies: + +Build: + +```bash +apt-get -y install \ + curl \ + build-essential \ + cmake \ + ccache \ + ninja-build \ + pkg-config \ + libcurl4-openssl-dev \ + libicu-dev \ + libturbojpeg0-dev \ + libpng-dev \ + libprotobuf-dev \ + libuv1-dev \ + libx11-dev \ + libegl-dev \ + libopengl-dev \ + xvfb \ + libjpeg-dev \ + libsqlite3-dev \ + libopengl0 \ + git +``` + +Runtime: + +```bash +apt install --no-install-recommends -y \ + xvfb \ + xauth \ + curl \ + libicu67 \ + libjpeg-turbo-progs \ + libpng16-16 \ + libprotobuf23 \ + libuv1 \ + libx11-6 \ + libegl1 \ + libopengl0 +``` ### PyBind11 diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.ubuntu20.04 similarity index 70% rename from docker/Dockerfile.dev rename to docker/Dockerfile.ubuntu20.04 index 5a6f6fe..759f862 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.ubuntu20.04 @@ -3,6 +3,9 @@ FROM ubuntu:20.04 WORKDIR /app ENV DISPLAY :99 +ENV DEBIAN_FRONTEND noninteractive +ENV PYTHONMALLOC malloc +ENV BUILD_TEMP_DIR /tmp/build RUN apt-get update && \ apt-get -y install software-properties-common && \ @@ -15,8 +18,10 @@ RUN apt-get update && \ ccache \ ninja-build \ pkg-config \ - python3.9 \ - python3.9-dev \ + # install python 3.9 + python3 \ + python3-dev \ + python3-pip \ # required dependencies libcurl4-openssl-dev \ libicu-dev \ @@ -31,13 +36,7 @@ RUN apt-get update && \ xvfb \ # debugging utilities valgrind && \ - # install pip for python3.9 - curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ - python3.9 get-pip.py && \ /usr/sbin/update-ccache-symlinks # Install python test deps -RUN pip3.9 install pytest pytest-benchmark pytest-valgrind python-dotenv Pillow numpy pixelmatch requests - -ENV BUILD_TEMP_DIR /tmp/build -ENV PYTHONMALLOC malloc \ No newline at end of file +RUN python3 -m pip install pytest pytest-benchmark pytest-valgrind python-dotenv Pillow numpy pixelmatch requests diff --git a/docker/Dockerfile.ubuntu22.04 b/docker/Dockerfile.ubuntu22.04 new file mode 100644 index 0000000..1481f8f --- /dev/null +++ b/docker/Dockerfile.ubuntu22.04 @@ -0,0 +1,43 @@ +FROM ubuntu:22.04 + +WORKDIR /app + +ENV DISPLAY :99 +ENV DEBIAN_FRONTEND noninteractive +ENV PYTHONMALLOC malloc +ENV BUILD_TEMP_DIR /tmp/build + +RUN apt-get update && \ + apt-get -y install software-properties-common && \ + add-apt-repository -y ppa:deadsnakes/ppa && \ + apt-get -y install \ + curl \ + # build tools + build-essential \ + cmake \ + ccache \ + ninja-build \ + pkg-config \ + # install python 3.10 + python3 \ + python3-dev \ + python3-pip \ + # required dependencies + libcurl4-openssl-dev \ + libicu-dev \ + libturbojpeg0-dev \ + libpng-dev \ + libprotobuf-dev \ + libuv1-dev \ + libx11-dev \ + libegl-dev \ + libopengl-dev \ + # runtime dependencies + xvfb \ + # debugging utilities + valgrind && \ + /usr/sbin/update-ccache-symlinks + +# Install python test deps +RUN python3 -m pip install pytest pytest-benchmark pytest-valgrind python-dotenv Pillow numpy pixelmatch requests + diff --git a/docker/README.md b/docker/README.md index 32659fd..0c0e147 100644 --- a/docker/README.md +++ b/docker/README.md @@ -5,20 +5,20 @@ Build the development container (using source files on host): ```bash -docker build -f docker/Dockerfile.dev -t pymgl-dev . +docker build -f docker/Dockerfile.ubuntu22.04 -t pymgl-dev-ubuntu22.04 . ``` Run the container mounted to the host filesystem for source files: ```bash -docker run -it -v "$PWD/:/app" pymgl-dev /bin/bash +docker run -it -v "$PWD/:/app" pymgl-dev-ubuntu22.04 /bin/bash ``` Then from within the container, run: ```bash -python3.9 setup.py build_ext --debug --inplace -python3.9 setup.py develop +python3 setup.py build_ext --debug --inplace +python3 setup.py develop # tests require virtual display Xvfb ${DISPLAY} -screen 0 "1024x768x24" -ac +render -noreset -nolisten tcp & @@ -71,5 +71,5 @@ docker cp :/tmp/valgrind-output /tmp/ To test for memory leaks using the Python tests, run: ```bash -valgrind --show-leak-kinds=definite --log-file=/tmp/valgrind-output python3.9 -m pytest pymgl/tests -vv --valgrind --valgrind-log=/tmp/valgrind-output > valgrind.log +valgrind --show-leak-kinds=definite --log-file=/tmp/valgrind-output python3 -m pytest pymgl/tests -vv --valgrind --valgrind-log=/tmp/valgrind-output > valgrind.log ``` diff --git a/pymgl/tests/fixtures/example-style-bad-source.json b/pymgl/tests/fixtures/example-style-bad-source.json index a5714e4..a912bc6 100644 --- a/pymgl/tests/fixtures/example-style-bad-source.json +++ b/pymgl/tests/fixtures/example-style-bad-source.json @@ -3,7 +3,7 @@ "sources": { "basemap": { "type": "raster", - "tiles": ["http://google.com/bad-endpoint/{z}/{x}/{y}.png"], + "tiles": ["https://google.com/bad-endpoint/{z}/{x}/{y}.png"], "tileSize": 256 }, "geojson": { diff --git a/pymgl/tests/test_style.py b/pymgl/tests/test_style.py index db36bec..d768ab4 100644 --- a/pymgl/tests/test_style.py +++ b/pymgl/tests/test_style.py @@ -157,7 +157,7 @@ def test_image_pattern(): img_data = map.renderPNG() - assert image_matches(img_data, f"{test}.png") + assert image_matches(img_data, f"{test}.png", tolerance=350) def test_bad_source(): diff --git a/tests/StyleTest.cpp b/tests/StyleTest.cpp index 4b7c1cc..55d9127 100644 --- a/tests/StyleTest.cpp +++ b/tests/StyleTest.cpp @@ -259,7 +259,7 @@ TEST(Style, ImagePattern) { // write_test_image(img, img_filename, true); write_test_image(img, img_filename, false); - EXPECT_TRUE(image_matches(img_filename, 10)); + EXPECT_TRUE(image_matches(img_filename, 350)); } // Tests of bad inputs