linux_distro_builds.yml

name: Linux Distro Builds 🐧

on:
  push:
    branches: ["main"]
  # No paths filter here: the required "Linux distro builds" check has to report
  # on every PR, and a workflow that never triggers leaves it pending forever.
  # `labeled` is here so the ci:distros escape hatch takes effect without a
  # push. Adding `types` replaces the defaults, so the three default activity
  # types have to be listed explicitly.
  pull_request:
    types: [opened, synchronize, reopened, labeled]
  merge_group:
  workflow_dispatch:

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name == 'pull_request' && 'pr' || github.run_id }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  build-linux:
    name: Build on ${{ matrix.distro }}
    # Label a PR `ci:distros` to opt it back in.
    if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci:distros')
    runs-on: ubuntu-latest
    container:
      image: ${{ matrix.image }}
    env:
      HF_TOKEN: ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }}
      CMAKE_C_COMPILER_LAUNCHER: ccache
      CMAKE_CXX_COMPILER_LAUNCHER: ccache
      CCACHE_DIR: ${{ github.workspace }}/.ccache

    strategy:
      fail-fast: ${{ github.event_name == 'merge_group' }}
      matrix:
        include:
          - distro: Arch
            image: archlinux:latest
            python: python
          - distro: openSUSE
            image: opensuse/tumbleweed:latest
            python: python3

    steps:
      - name: Checkout code
        uses: actions/checkout@v5
        with:
          clean: true
          fetch-depth: 0

      - name: Cache package downloads and ccache
        uses: actions/cache@v5
        with:
          path: |
            .package-cache
            .ccache
          key: cache-linux-distro-${{ matrix.distro }}-${{ hashFiles('CMakeLists.txt') }}-${{ github.run_id }}-${{ github.run_attempt }}
          restore-keys: |
            cache-linux-distro-${{ matrix.distro }}-${{ hashFiles('CMakeLists.txt') }}-
            cache-linux-distro-${{ matrix.distro }}-

      - name: Setup package cache directories
        run: |
          mkdir -p .package-cache/pacman .package-cache/zypp .package-cache/apt .package-cache/dnf
          if command -v pacman >/dev/null 2>&1; then
            mkdir -p /var/cache/pacman
            rm -rf /var/cache/pacman/pkg
            ln -s "$GITHUB_WORKSPACE/.package-cache/pacman" /var/cache/pacman/pkg
          elif command -v zypper >/dev/null 2>&1; then
            mkdir -p /var/cache
            rm -rf /var/cache/zypp
            ln -s "$GITHUB_WORKSPACE/.package-cache/zypp" /var/cache/zypp
          elif command -v apt-get >/dev/null 2>&1; then
            mkdir -p /var/cache/apt
            rm -rf /var/cache/apt/archives
            ln -s "$GITHUB_WORKSPACE/.package-cache/apt" /var/cache/apt/archives
          elif command -v dnf >/dev/null 2>&1; then
            mkdir -p /var/cache
            rm -rf /var/cache/dnf
            ln -s "$GITHUB_WORKSPACE/.package-cache/dnf" /var/cache/dnf
          fi

      - name: Install ccache
        run: |
          if command -v pacman >/dev/null 2>&1; then
            pacman -Syu --needed --noconfirm ccache
          elif command -v zypper >/dev/null 2>&1; then
            zypper install -y ccache
          elif command -v apt-get >/dev/null 2>&1; then
            apt-get update && apt-get install -y ccache
          elif command -v dnf >/dev/null 2>&1; then
            dnf install -y ccache
          fi
          ccache -z

      - name: Run setup.sh to configure environment
        run: |
          set -e

          echo "Running setup.sh for ${{ matrix.distro }}..."
          bash setup.sh

          echo "Verifying process management tools..."
          command -v pgrep && command -v pkill && command -v ps

          echo "Verifying SSL certificates..."
          ls -la /etc/ssl/certs/ | head -10

          echo "Testing HTTPS connectivity..."
          curl -sI https://huggingface.co | head -5 || echo "Warning: HTTPS test failed"

          echo "Setup completed successfully!"

      - name: Build C++ Server with CMake
        run: |
          set -e

          echo "Building lemond and lemonade on ${{ matrix.distro }}..."

          # setup.sh configures with testing on and the build below has no
          # --target, so every unit-test binary would be built and then left out
          # of the package. Same reasoning as BUILD_TESTING=OFF in
          # contrib/debian/rules and the RPM job.
          echo "Reconfiguring without the unit-test targets..."
          cmake --preset default -DBUILD_TESTING=OFF

          # Build
          echo "Building binaries..."
          cmake --build --preset default

          # Verify binaries exist
          if [ ! -f "build/lemond" ]; then
              echo "ERROR: lemond not found!"
              echo "Build directory contents:"
              ls -lh build/
              exit 1
          fi

          if [ ! -f "build/lemonade" ]; then
              echo "ERROR: lemonade not found!"
              echo "Build directory contents:"
              ls -lh build/
              exit 1
          fi

          echo "Binaries found successfully"
          ls -lh build/lemond build/lemonade

          echo "Verifying binary executability..."
          ./build/lemond --version
          ./build/lemonade --version

          echo "=== ccache statistics ==="
          ccache -s

          echo "${{ matrix.distro }} build successful!"

      - name: Verify system library linking
        run: |
          echo "Checking library dependencies..."
          cd build

          echo "=== lemond dependencies ==="
          ldd lemond

          echo ""
          echo "=== lemonade dependencies ==="
          ldd lemonade

          echo ""
          echo "All dependencies resolved successfully on ${{ matrix.distro }}!"

      - name: Setup Python virtual environment
        run: |
          echo "Creating Python virtual environment..."
          if [ "${{ matrix.distro }}" = "Debian" ]; then
              apt install -y python3-venv
          fi

          ${{ matrix.python }} -m venv .venv

          echo "Activating virtual environment and installing dependencies..."
          . .venv/bin/activate
          pip install --upgrade pip
          pip install -r test/requirements.txt

          echo "Python environment setup complete!"

      - name: Start server
        env:
          LEMONADE_CI_MODE: "True"
        run: |
          echo "Starting lemond in background..."
          mkdir -p "$RUNNER_TEMP/xdg-runtime"
          chmod 700 "$RUNNER_TEMP/xdg-runtime"
          echo "XDG_RUNTIME_DIR=$RUNNER_TEMP/xdg-runtime" >> "$GITHUB_ENV"
          export XDG_RUNTIME_DIR="$RUNNER_TEMP/xdg-runtime"
          ./build/lemond --port 13305 > "$RUNNER_TEMP/lemonade-server.log" 2>&1 &

          echo "Waiting for server to be ready..."
          for i in $(seq 1 30); do
            if curl -sf http://localhost:13305/live > /dev/null 2>&1; then
              echo "Server is running and healthy"
              exit 0
            fi
            sleep 2
          done
          echo "ERROR: Server did not start"
          cat "$RUNNER_TEMP/lemonade-server.log" 2>/dev/null || true
          exit 1

      - name: Run CLI tests
        env:
          LEMONADE_CI_MODE: "True"
          PYTHONIOENCODING: utf-8
        run: |
          set -e

          . .venv/bin/activate
          CLI_BINARY="$(pwd)/build/lemonade"

          echo "Running CLI tests..."
          python test/server_cli2.py --cli-binary "$CLI_BINARY"

          echo "CLI tests PASSED!"

      - name: Run endpoint tests
        env:
          LEMONADE_CI_MODE: "True"
          PYTHONIOENCODING: utf-8
        run: |
          set -e

          . .venv/bin/activate
          CLI_BINARY="$(pwd)/build/lemonade"

          echo "Running endpoint tests..."
          python test/server_endpoints.py --cli-binary "$CLI_BINARY"

          echo "Endpoint tests PASSED!"

      - name: Run log rotation tests
        env:
          LEMONADE_CI_MODE: "True"
          PYTHONIOENCODING: utf-8
        run: |
          set -e

          . .venv/bin/activate
          CLI_BINARY="$(pwd)/build/lemonade"

          echo "Running log rotation tests..."
          python test/server_log_rotation.py

          echo "Log rotation tests PASSED!"

      - name: Capture and upload server logs
        if: always()
        uses: ./.github/actions/capture-server-logs
        with:
          artifact-name: server-logs-distro-${{ matrix.distro }}

  # Gate job that ensures: 1. in the merge queue, all jobs in `needs:` ran
  # successfully (otherwise the merge is blocked), and 2. these jobs do not need
  # to run on ordinary pull request pushes.
  distro-gate:
    name: Linux distro builds
    needs: [build-linux]
    if: always()
    runs-on: ubuntu-latest
    steps:
      - name: Check gated jobs
        env:
          NEEDS: ${{ toJSON(needs) }}
        run: |
          # $NEEDS: {"job-id": {"result": "success|failure|skipped|cancelled"}, ...}
          # Fail if any job broke. In the merge queue, also fail if any never ran.
          echo "$NEEDS"
          broke=$(jq -r 'to_entries[]|select(.value.result=="failure" or .value.result=="cancelled")|.key' <<<"$NEEDS")
          if [ -n "$broke" ]; then
            echo "FAILED: $broke"
            exit 1
          fi
          if [ "${{ github.event_name }}" = "merge_group" ]; then
            absent=$(jq -r 'to_entries[]|select(.value.result!="success")|.key' <<<"$NEEDS")
            if [ -n "$absent" ]; then
              echo "DID NOT RUN IN MERGE QUEUE: $absent"
              exit 1
            fi
          fi