gpu_skyrl_train.yaml

name: SkyRL-Train-GPU

on: 
  push: 
    branches: 
      - main
    paths:
      - 'ci/**'
      - 'skyrl/backends/skyrl_train/**'
      - 'skyrl/train/**'
      - 'tests/backends/skyrl_train/**'
      - 'pyproject.toml'
      - '!docs/**'
      - '!examples/**'
      - '.github/workflows/**'
  pull_request_target:
    types: [labeled]
  workflow_dispatch:


permissions:
  checks: write   # for status checks to appear
  contents: read

jobs:
  
  skyrl_train_tests:
    # Gate on the label that fired this event. `types: [labeled]` delivers one event
    # per label added, and `contains(...labels.*.name, ...)` tests the whole label
    # set, so labelling in bulk launched one identical GPU run per label.
    if: >
      github.event_name == 'push' ||
      github.event_name == 'workflow_dispatch' ||
      (
        github.event_name == 'pull_request_target' &&
        !github.event.pull_request.draft &&
        github.event.label.name == 'run_train_gpu_ci'
      )
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash
        working-directory: .

    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha || github.ref }}
          # The Anyscale job uploads this working dir, so we need the PR head.
          # Fork code only reaches here after a maintainer applies the gating
          # label above; don't leave the base repo token on disk for it.
          allow-unsafe-pr-checkout: true
          persist-credentials: false
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.12'
      - name: Install the latest version of uv
        uses: astral-sh/setup-uv@v6
        with:
          activate-environment: true
      - name: Install basic dependencies
        run: uv pip install anyscale==0.26.103 typer==0.9.0
      - name: Install envsubst
        run: sudo apt-get update && sudo apt-get install -y gettext-base
      # Run tests
      - name: GPU tests
        env:
          ANYSCALE_CLI_TOKEN: ${{ secrets.ANYSCALE_CLI_TOKEN }}
          ANYSCALE_HOST: https://console.anyscale.com
          HF_TOKEN: ${{ secrets.HF_TOKEN }}
        run: |
          envsubst < ci/anyscale_gpu_ci_skyrl_train.yaml > ci/anyscale_gpu_ci_skyrl_train_envsubst.yaml
          COMMIT_SHA="${{ github.event.pull_request.head.sha || github.sha }}"
          JOB_NAME="skyrl-train-gpu-ci-${COMMIT_SHA:0:7}-${{ github.run_id }}-${{ github.run_attempt }}"
          bash ci/submit_anyscale_job.sh ci/anyscale_gpu_ci_skyrl_train_envsubst.yaml "$JOB_NAME" 12000
          rm -f ci/anyscale_gpu_ci_skyrl_train_envsubst.yaml
      # Cancelling the run stops the runner, not the cluster. GitHub still runs
      # `if: cancelled()` steps, so this is where an orphaned GPU job gets reaped.
      - name: Terminate Anyscale jobs on cancellation
        if: cancelled()
        env:
          ANYSCALE_CLI_TOKEN: ${{ secrets.ANYSCALE_CLI_TOKEN }}
          ANYSCALE_HOST: https://console.anyscale.com
        run: bash ci/terminate_anyscale_jobs.sh