release_python.yml

# Release workflow for the Kaskada Python library.
#
# This flow is triggered by the creation of a new GitHub release.
# Generally, the new release should be marked as a "Pre-Release".
#
# The creation of the release will cause this to determine the version
# and build the Python wheels on multiple platforms, before combining
# them and uploading the new Python library to PyPI.
#
# Ideas for improvement
# ---------------------
# 1. After releasing we should update the `latest` tag. We don't do this
#    yet because the old version of the Kaskada Python client downloads
#    the latest engine release.
# 2. After releasing, we should update the version in the checked in
#    project configurations to be a `-dev` pre-release of the next version.
# 3. We could add a step that uploads the wheels to Test PyPi for verification.
# 4. We could build the wheels on merge to main, allowing the release process
#    to retrieve them rather than building them. This would also let us do the
#    the bulk of the multi-platform testing regularly, as well as making it
#    less likely that we'd need to recreate a release to deal with problems.
# 5. Schedule a weekly, automatic release.
name: Python Release

# Only one job per-ref
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

on:
  release:
    types:
      - published

defaults:
  run:
    shell: bash
    working-directory: python

permissions:
  contents: read

jobs:
  version:
    runs-on: ubuntu-latest
    outputs:
      version: ${{ steps.set-version.outputs.version }}
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: 3.11
      - name: Determine version (current)
        if: github.event_name != 'release'
        run: |
          pip install tomlkit packaging
          VERSION=$(python ../scripts/version.py get pyproject.toml project.version)
          echo "VERSION=$VERSION" >> $GITHUB_ENV
      - name: Determine version (release)
        if: github.event_name == 'release'
        run: |
          echo "GITHUB_REF=${GITHUB_REF}"
          VERSION=${GITHUB_REF#refs/tags/v}
          pip install tomlkit packaging
          echo "VERSION=${VERSION}"
          echo "VERSION=$VERSION" >> $GITHUB_ENV
      - name: Set version
        id: set-version
        run: |
          echo "version=$VERSION" >> $GITHUB_OUTPUT

  build-wheel-macos:
    # Build wheels during release.
    if: github.event_name == 'release'
    runs-on: macos-latest
    needs: [version]
    strategy:
      matrix:
        include:
          - target: x86_64
            wheel_suffix: macosx_11_0_x86_64
          # - target: arm64
          #   wheel_suffix: macosx_11_0_arm64
          - target: universal2
            wheel_suffix: macosx_11_0_x86_64.macosx_11_0_arm64.macosx_11_0_universal2
    steps:
      - uses: actions/checkout@v3
      - name: Install poetry
        run: |
          pipx install poetry
          poetry config virtualenvs.create true --local
          poetry config virtualenvs.in-project true --local
      - uses: actions/setup-python@v4
        with:
          python-version: |
            3.9
            3.10
            3.11
          architecture: x64
          cache: poetry
      - name: Install Protoc
        uses: arduino/setup-protoc@v1
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Set Version (For Release)
        if: github.event_name == 'release'
        run: |
          pip install tomlkit packaging
          python ../scripts/version.py set ${{ needs.version.outputs.version }} \
            pyproject.toml:project.version \
            pyproject.toml:tool.poetry.version \
            Cargo.toml:package.version
      - name: Build wheel
        uses: messense/maturin-action@v1
        with:
          target: ${{ matrix.target }}
          args: --release --out dist --sdist
          working-directory: python
      - name: pytest and mypy
        if: matrix.target != 'arm64'
        run: |
          pip install tomlkit packaging
          VERSION=$(python ../scripts/version.py normalize ${{ needs.version.outputs.version }})
          WHEEL="dist/kaskada-${VERSION}-cp38-abi3-${{ matrix.wheel_suffix }}.whl"
          echo "WHEEL:${WHEEL}"
          for V in 3.9 3.10 3.11; do
            echo "::group::Install for Python $V"
            poetry env use $V
            source $(poetry env info --path)/bin/activate
            poetry install --only=test --only=typecheck
            pip install ${WHEEL} --force-reinstall
            echo "::endgroup::"
            echo "::group::Test Python $V"
            poetry run pytest
            echo "::endgroup::"
            echo "::group::MyPy Python $V"
            poetry run mypy -- --install-types --non-interactive pysrc pytests
            echo "::endgroup::"
            deactivate
          done
      - name: Upload wheels
        uses: actions/upload-artifact@v2
        with:
          name: wheels
          path: ${{ github.workspace }}/python/dist

  build-wheel-windows:
    if: github.event_name == 'release'
    needs: [version]
    runs-on: windows-latest
    steps:
      - uses: actions/checkout@v3
      - name: Install poetry
        run: |
          pipx install poetry
          poetry config virtualenvs.create true --local
          poetry config virtualenvs.in-project true --local
      - uses: actions/setup-python@v4
        with:
          python-version: 3.11
          architecture: x64
          cache: poetry
      - name: Install Protoc
        uses: arduino/setup-protoc@v1
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Set Version (For Release)
        if: github.event_name == 'release'
        run: |
            pip install tomlkit packaging
            python ../scripts/version.py set ${{ needs.version.outputs.version }} \
              pyproject.toml:project.version \
              pyproject.toml:tool.poetry.version \
              Cargo.toml:package.version
      - name: Build wheels
        uses: messense/maturin-action@v1
        with:
          target: x64
          args: --release --out dist
          working-directory: python
      - name: pytest and mypy (Windows x64)
        shell: bash
        run: |
          echo "::group::Install for Python 3.11"
          source $(poetry env info --path)\\Scripts\\activate
          poetry install --only=test --only=typecheck
          pip install tomlkit packaging
          VERSION=$(python ../scripts/version.py normalize ${{ needs.version.outputs.version }})
          WHEEL="dist/kaskada-${VERSION}-cp38-abi3-win_amd64.whl"
          echo "WHEEL:${WHEEL}"
          pip install ${WHEEL} --force-reinstall
          echo "::endgroup::"
          echo "::group::Test Python 3.11"
          poetry run pytest
          echo "::endgroup::"
          echo "::group::MyPy Python 3.11"
          poetry run mypy -- --install-types --non-interactive pysrc pytests
          echo "::endgroup::"
          deactivate
      - name: Upload wheels
        uses: actions/upload-artifact@v2
        with:
          name: wheels
          path: ${{ github.workspace }}/python/dist

  build-wheel-linux:
    if: github.event_name == 'release'
    needs: [version]
    runs-on: ubuntu-latest
    strategy:
      matrix:
        include:
          - target: x86_64
    steps:
      - uses: actions/checkout@v3
      - name: Install poetry
        run: |
          pipx install poetry
          poetry config virtualenvs.create true --local
          poetry config virtualenvs.in-project true --local
      - uses: actions/setup-python@v4
        with:
          python-version: |
            3.9
            3.10
            3.11
          cache: poetry
      - name: Set Version (For Release)
        if: github.event_name == 'release'
        run: |
          pip install tomlkit packaging
          python ../scripts/version.py set ${{ needs.version.outputs.version }} \
            pyproject.toml:project.version \
            pyproject.toml:tool.poetry.version \
            Cargo.toml:package.version
      - name: Build wheels
        uses: messense/maturin-action@v1
        with:
          target: ${{ matrix.target }}
          manylinux: 2_28
          args: --release --out dist
          working-directory: python
          before-script-linux: |
            set -e
            dnf -y install clang protobuf-devel lld
            clang --version
            protoc --version
      - name: pytest and mypy (Linux x86_64)
        if: matrix.target == 'x86_64'
        run: |
          pip install tomlkit packaging
          VERSION=$(python ../scripts/version.py normalize ${{ needs.version.outputs.version }})
          WHEEL="dist/kaskada-${VERSION}-cp38-abi3-manylinux_2_28_${{ matrix.target }}.whl"
          echo "WHEEL:${WHEEL}"
          for V in 3.9 3.10 3.11; do
            echo "::group::Install for Python $V"
            poetry env use $V
            poetry env info
            source $(poetry env info --path)/bin/activate
            poetry install --only=test --only=typecheck
            pip install ${WHEEL} --force-reinstall
            echo "::endgroup::"
            echo "::group::Test Python $V"
            poetry run pytest
            echo "::endgroup::"
            echo "::group::MyPy Python $V"
            poetry run mypy -- --install-types --non-interactive pysrc pytests
            echo "::endgroup::"
            deactivate
          done
      - name: Upload wheels
        uses: actions/upload-artifact@v2
        with:
          name: wheels
          path: ${{ github.workspace }}/python/dist

  # Make the source distribution
  sdist:
    if: github.event_name == 'release'
    needs: [version]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Set Version (For Release)
        if: github.event_name == 'release'
        run: |
          pip install tomlkit packaging
          python ../scripts/version.py set ${{ needs.version.outputs.version }} \
            pyproject.toml:project.version \
            pyproject.toml:tool.poetry.version \
            Cargo.toml:package.version
      - name: Build sdist
        uses: PyO3/maturin-action@v1
        with:
          command: sdist
          args: --out dist
          working-directory: python
      - name: Upload sdist
        uses: actions/upload-artifact@v3
        with:
          name: wheels
          path: ${{ github.workspace }}/python/dist

  release:
    name: Release
    runs-on: ubuntu-latest
    environment: pypi
    needs: [build-wheel-linux, build-wheel-windows, build-wheel-macos, sdist]
    permissions:
      # Fetch the OpenID Connect token used to publish to PyPi.
      id-token: write
      # For deploying release artifacts.
      contents: write
      # For creating the release announcement.
      discussions: write
    if: github.event_name == 'release' && github.event.action == 'published'
    steps:
      - uses: actions/checkout@v3
      - uses: actions/download-artifact@v3
        with:
          name: wheels
          path: dist

      - uses: pypa/gh-action-pypi-publish@release/v1
        # with:
        #   repository-url: https://test.pypi.org/legacy/

      - name: Publish release
        # We don't need the source for the release, so we don't currently check it out.
        # However, we set a default working-directory of `python` which is within the source.
        # To avoid problems, we explicitly set the working-directory to the root here.
        working-directory:
        run: |
          gh release edit ${{ github.ref_name }} \
            --draft=false --latest --discussion-category "Announcements"
        env:
          GH_TOKEN: ${{ github.token }}