OptiTrack (2/3): NatNet server emulator + host integration tests #71
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: System Tests | |
| on: | |
| pull_request: | |
| types: [opened] | |
| issue_comment: | |
| types: [created] | |
| workflow_dispatch: | |
| inputs: | |
| marks: | |
| description: "pytest marks expression (e.g. 'build_docker', 'liveliness', 'takeoff_hover_land'). \ | |
| Use 'or' to combine marks: 'liveliness or takeoff_hover_land'. Leave blank to run all marks. \ | |
| Note: 'build_packages' is automatically prepended whenever any marks are specified, \ | |
| to ensure code is built before launch tests run." | |
| default: "liveliness or takeoff_hover_land" | |
| required: false | |
| sim: | |
| description: "Sim targets, comma-separated: msairsim,isaacsim" | |
| default: msairsim,isaacsim | |
| required: false | |
| num_robots: | |
| description: "Robot counts, comma-separated (e.g. 1,3)" | |
| default: "1" | |
| required: false | |
| stress_iterations: | |
| description: "Iterations per (sim, num_robots) config" | |
| default: "1" | |
| required: false | |
| stable_duration: | |
| description: "Seconds for test_stable polling window" | |
| default: "120" | |
| required: false | |
| baseline_run_id: | |
| description: "Run ID to use as baseline for metric comparison (blank = latest successful run on main)" | |
| default: "" | |
| required: false | |
| jobs: | |
| run-tests: | |
| name: Run Tests | |
| runs-on: [self-hosted, airstack-ephemeral] | |
| # Triggers: | |
| # - workflow_dispatch (manual) | |
| # - PR opened from the same repo (not a fork) — same-repo guard | |
| # prevents arbitrary code execution on the self-hosted runner from | |
| # untrusted contributors. | |
| # - PR comment starting with `/pytest` from a user with write access | |
| # (OWNER/MEMBER/COLLABORATOR). issue_comment fires for both issues | |
| # and PRs; `issue.pull_request` disambiguates. The author_association | |
| # gate is what keeps random commenters from running code on the | |
| # self-hosted runner. | |
| if: | | |
| github.event_name == 'workflow_dispatch' || | |
| (github.event_name == 'pull_request' && | |
| github.event.pull_request.head.repo.full_name == github.repository) || | |
| (github.event_name == 'issue_comment' && | |
| github.event.issue.pull_request != null && | |
| startsWith(github.event.comment.body, '/pytest') && | |
| contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.comment.author_association)) | |
| timeout-minutes: 120 | |
| # Adding any `permissions:` entry disables GITHUB_TOKEN's defaults, so | |
| # every scope used here has to be re-granted explicitly: | |
| # checks:write — create/update the Check Run on the PR head | |
| # contents:read — actions/checkout | |
| # pull-requests:write — post the acknowledgment comment on the PR. | |
| # Even though the endpoint is /issues/{n}/comments, comments on | |
| # PRs are gated by the pull-requests permission, not issues — the | |
| # `x-accepted-github-permissions` header lists both as alternatives | |
| # but only pull-requests:write actually works for PR comments. | |
| permissions: | |
| checks: write | |
| contents: read | |
| pull-requests: write | |
| # Mirror the registry password into env so step-level `if:` expressions | |
| # can check whether registry-cache mode is available — `secrets.*` itself | |
| # is not addressable from `if:` expressions. | |
| env: | |
| DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }} | |
| steps: | |
| # Uses actions/github-script (Node, bundled with the runner) instead | |
| # of `gh` so we don't depend on system tools — the ephemeral | |
| # self-hosted runner doesn't have gh/jq installed. | |
| - name: Resolve PR head | |
| if: github.event_name == 'issue_comment' | |
| id: pr | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const pr = await github.rest.pulls.get({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| pull_number: context.issue.number, | |
| }); | |
| // Even though the commenter has write access, the PR's code | |
| // lives on the head repo. If that's a fork, we'd be running | |
| // untrusted code on the self-hosted runner. | |
| const headRepo = pr.data.head.repo.full_name; | |
| const expected = `${context.repo.owner}/${context.repo.repo}`; | |
| if (headRepo !== expected) { | |
| core.setFailed(`PR #${context.issue.number} is from a fork (${headRepo}); /pytest is not supported for forks.`); | |
| return; | |
| } | |
| core.setOutput('head_sha', pr.data.head.sha); | |
| core.setOutput('base_ref', pr.data.base.ref); | |
| # Parsed up-front (before checkout) so the acknowledgment comment below | |
| # can echo the resolved args. This step only reads env vars, so it | |
| # doesn't need the working tree. | |
| - name: Parse pytest args | |
| id: parse | |
| env: | |
| GH_EVENT_NAME: ${{ github.event_name }} | |
| COMMENT_BODY: ${{ github.event.comment.body }} | |
| INPUT_MARKS: ${{ inputs.marks }} | |
| INPUT_SIM: ${{ inputs.sim }} | |
| INPUT_NUM_ROBOTS: ${{ inputs.num_robots }} | |
| INPUT_ITERATIONS: ${{ inputs.stress_iterations }} | |
| INPUT_STABLE: ${{ inputs.stable_duration }} | |
| run: | | |
| python3 <<'PYEOF' | |
| import os, shlex, sys | |
| event = os.environ['GH_EVENT_NAME'] | |
| if event == 'workflow_dispatch': | |
| args = [] | |
| if (m := os.environ.get('INPUT_MARKS', '').strip()): | |
| args.extend(['-m', m]) | |
| if (s := os.environ.get('INPUT_SIM', '').strip()): | |
| args.extend(['--sim', s]) | |
| if (n := os.environ.get('INPUT_NUM_ROBOTS', '').strip()): | |
| args.extend(['--num-robots', n]) | |
| if (it := os.environ.get('INPUT_ITERATIONS', '').strip()): | |
| args.extend(['--stress-iterations', it]) | |
| if (st := os.environ.get('INPUT_STABLE', '').strip()): | |
| args.extend(['--stable-duration', st]) | |
| elif event == 'pull_request': | |
| # PR-opened auto-run uses pytest's conftest defaults — same as | |
| # /pytest with no args. | |
| args = [] | |
| else: | |
| body = os.environ.get('COMMENT_BODY', '') | |
| # Only the first line is parsed — everything below it is | |
| # treated as freeform comment text (notes, context, etc.). | |
| first_line = (body.splitlines()[0] if body else '').strip() | |
| if not first_line.startswith('/pytest'): | |
| print('::error::Comment does not start with /pytest', file=sys.stderr) | |
| sys.exit(1) | |
| args_line = first_line[len('/pytest'):].strip() | |
| try: | |
| args = shlex.split(args_line) | |
| except ValueError as e: | |
| print(f'::error::Could not parse pytest args from comment: {e}', file=sys.stderr) | |
| sys.exit(1) | |
| # Pull out --sim and -m so the image-prep step can scope profiles | |
| # and decide whether to skip (build_docker tests rebuild themselves). | |
| # When --sim isn't given we mirror conftest's default so prep covers | |
| # whatever pytest will actually exercise. | |
| sim = 'msairsim,isaacsim' | |
| marks = '' | |
| marks_idx = -1 | |
| for i, a in enumerate(args): | |
| if a == '--sim' and i + 1 < len(args): | |
| sim = args[i + 1] | |
| elif a == '-m' and i + 1 < len(args): | |
| marks = args[i + 1] | |
| marks_idx = i + 1 | |
| # When the user specified any marks, prepend build_packages so code | |
| # is built before launch tests try to use it. Skipped when no marks | |
| # are given (pytest runs everything including build_packages) and | |
| # when build_packages is already in the expression. | |
| if marks and 'build_packages' not in marks: | |
| marks = f'build_packages or {marks}' | |
| args[marks_idx] = marks | |
| skip_prep = 'build_docker' in marks | |
| quoted = ' '.join(shlex.quote(a) for a in args) | |
| with open(os.environ['GITHUB_OUTPUT'], 'a') as f: | |
| f.write(f'pytest_args={quoted}\n') | |
| f.write(f'sim={sim}\n') | |
| f.write(f'skip_image_prep={"true" if skip_prep else "false"}\n') | |
| print(f'Resolved pytest args: {quoted or "(none — pytest defaults)"}') | |
| print(f'Resolved sim profile: {sim}') | |
| print(f'Skip image prep: {skip_prep}') | |
| PYEOF | |
| # Reply on the PR thread so the commenter sees their /pytest was | |
| # picked up and can confirm we parsed the args correctly. The | |
| # workflow_dispatch path skips this (no PR to comment on); the | |
| # pull_request-opened path skips it too (the PR Checks tab is | |
| # already showing the native run). | |
| - name: Post acknowledgment comment | |
| if: github.event_name == 'issue_comment' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const args = ${{ toJSON(steps.parse.outputs.pytest_args) }}; | |
| const cmd = `pytest tests/ ${args}`.trim(); | |
| const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; | |
| const note = `Note: \`build_packages\` is automatically prepended whenever any marks are specified, to ensure code is built before launch tests run.`; | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| body: `Running \`${cmd}\` — [view run](${runUrl}). Status will appear as a check on this PR.\n\n${note}`, | |
| }); | |
| # Pin a Check Run on the PR's head SHA so the run shows up in the | |
| # PR's "Checks" tab while it executes — issue_comment-triggered runs | |
| # are otherwise associated with the default branch and don't surface | |
| # on the PR. Finalized at end of job with the actual conclusion. | |
| - name: Open in-progress check on PR head | |
| if: github.event_name == 'issue_comment' | |
| id: check_create | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const res = await github.rest.checks.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| name: 'System Tests', | |
| head_sha: '${{ steps.pr.outputs.head_sha }}', | |
| status: 'in_progress', | |
| details_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`, | |
| }); | |
| core.setOutput('id', res.data.id); | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| # For issue_comment we must check out the PR head explicitly — | |
| # GITHUB_SHA points at the default branch for that event. For | |
| # workflow_dispatch and pull_request, empty string lets checkout | |
| # use its default (the dispatched ref / the PR merge commit). | |
| ref: ${{ github.event_name == 'issue_comment' && steps.pr.outputs.head_sha || '' }} | |
| submodules: recursive | |
| - name: Create Isaac Sim omni_pass.env | |
| run: | | |
| mkdir -p simulation/isaac-sim/docker | |
| cat > simulation/isaac-sim/docker/omni_pass.env <<'EOF' | |
| OMNI_USER=guest | |
| OMNI_PASS=guest | |
| OMNI_SERVER="omniverse://airlab-nucleus.andrew.cmu.edu/NVIDIA/Assets/Isaac/5.1" | |
| ACCEPT_EULA=Y | |
| OMNI_ENV_PRIVACY_CONSENT=Y | |
| EOF | |
| - name: Install test dependencies | |
| # Ubuntu 24.04 marks the system Python as externally-managed (PEP 668), | |
| # so `pip install` outside a venv is rejected. Use a venv and prepend | |
| # its bin/ to $GITHUB_PATH so subsequent steps pick up `pytest` | |
| # automatically. | |
| run: | | |
| sudo apt-get update -qq | |
| sudo apt-get install -y --no-install-recommends python3-venv | |
| python3 -m venv .venv | |
| echo "$GITHUB_WORKSPACE/.venv/bin" >> "$GITHUB_PATH" | |
| .venv/bin/pip install --upgrade pip | |
| .venv/bin/pip install -r tests/requirements.txt | |
| # Optional registry-cache mode. When the secrets/vars are present we log | |
| # in to the internal Docker registry; the next step then sets | |
| # AIRSTACK_REGISTRY_CACHE=1 so airstack.sh pre-pulls + uses BuildKit | |
| # inline cache (build_docker tests get layer-reuse speedup) and pre-pulls | |
| # before `airstack up` (other tests skip the implicit rebuild). When | |
| # secrets are absent both steps are skipped and behavior is unchanged. | |
| - name: Log in to internal Docker registry | |
| id: docker_login | |
| if: ${{ vars.DOCKER_REGISTRY_URL != '' && env.DOCKER_REGISTRY_PASSWORD != '' }} | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ vars.DOCKER_REGISTRY_URL }} | |
| username: ${{ vars.DOCKER_REGISTRY_USERNAME }} | |
| password: ${{ secrets.DOCKER_REGISTRY_PASSWORD }} | |
| - name: Enable registry-cache mode | |
| if: ${{ steps.docker_login.outcome == 'success' }} | |
| run: echo "AIRSTACK_REGISTRY_CACHE=1" >> "$GITHUB_ENV" | |
| - name: Ensure airstack.sh is executable | |
| run: chmod +x airstack.sh | |
| # The ephemeral runner starts with no local images. `airstack_env` in | |
| # tests/conftest.py fails fast if compose images are missing, so prep | |
| # them here. Profile-gated services (ms-airsim, isaac-sim) are skipped | |
| # by compose unless their profile is active, so we mirror the fixture's | |
| # profile selection from the parsed --sim. Pull-only by default; fall | |
| # back to a full build only if the registry doesn't have everything | |
| # (e.g. new branch with no published image yet). Skipped when the | |
| # marks expression contains build_docker — those tests build per-service | |
| # themselves. | |
| - name: Ensure Docker images present | |
| if: ${{ steps.parse.outputs.skip_image_prep != 'true' }} | |
| env: | |
| AIRSTACK_ROOT: ${{ github.workspace }} | |
| SIM_INPUT: ${{ steps.parse.outputs.sim }} | |
| run: | | |
| profiles=desktop | |
| [[ ",$SIM_INPUT," == *,msairsim,* ]] && profiles="$profiles,ms-airsim" | |
| [[ ",$SIM_INPUT," == *,isaacsim,* ]] && profiles="$profiles,isaac-sim" | |
| export COMPOSE_PROFILES="$profiles" | |
| echo "Pulling images for COMPOSE_PROFILES=$COMPOSE_PROFILES" | |
| # Pull from registry; tolerate per-image failures so we can detect | |
| # what's still missing afterwards instead of aborting on the first | |
| # gap. `--progress=quiet` suppresses per-layer progress; errors | |
| # still surface on stderr. | |
| ./airstack.sh --progress=quiet image-pull --ignore-pull-failures || true | |
| missing=() | |
| while IFS= read -r img; do | |
| [[ -z "$img" ]] && continue | |
| if ! docker image inspect "$img" --format '{{.Id}}' >/dev/null 2>&1; then | |
| missing+=("$img") | |
| fi | |
| done < <(docker compose -f docker-compose.yaml config --images) | |
| if (( ${#missing[@]} > 0 )); then | |
| echo "Pull did not produce these images; falling back to build:" | |
| printf ' - %s\n' "${missing[@]}" | |
| ./airstack.sh --progress=quiet image-build | |
| else | |
| echo "All required images present after pull — skipping build." | |
| fi | |
| - name: Run tests | |
| env: | |
| AIRSTACK_ROOT: ${{ github.workspace }} | |
| DISPLAY: "" | |
| PYTEST_ARGS: ${{ steps.parse.outputs.pytest_args }} | |
| run: | | |
| # Re-split the shell-quoted args from the parse step so we forward | |
| # them to pytest as a proper argv list (preserving values like | |
| # `-m 'a or b'`). Empty PYTEST_ARGS yields an empty array, so | |
| # pytest falls back to its conftest defaults. | |
| mapfile -t ARGS < <(python3 -c "import os, shlex; print('\n'.join(shlex.split(os.environ['PYTEST_ARGS'])))") | |
| pytest tests/ \ | |
| "${ARGS[@]}" \ | |
| -v -s \ | |
| --log-cli-level=INFO \ | |
| --log-cli-format='%(asctime)s [%(levelname)s] %(name)s: %(message)s' \ | |
| --log-cli-date-format='%H:%M:%S' | |
| - name: Upload test results | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: test-results-${{ github.sha }}-${{ github.run_id }} | |
| path: tests/results/ | |
| retention-days: 90 | |
| # Close out the Check Run with the job's final conclusion. The | |
| # `steps.check_create.outputs.id` guard skips this when the open | |
| # step didn't run (workflow_dispatch) or failed before producing | |
| # an id. | |
| - name: Finalize check on PR head | |
| if: always() && github.event_name == 'issue_comment' && steps.check_create.outputs.id | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| await github.rest.checks.update({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| check_run_id: ${{ steps.check_create.outputs.id }}, | |
| status: 'completed', | |
| conclusion: '${{ job.status }}', | |
| details_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`, | |
| }); | |
| report: | |
| name: Metrics Report | |
| runs-on: ubuntu-latest | |
| needs: run-tests | |
| # Skip when run-tests was skipped (e.g., comment didn't match `/pytest`) | |
| # so we don't post empty-report comments on every PR comment. | |
| if: always() && needs.run-tests.result != 'skipped' | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Install report dependencies | |
| run: pip install tabulate | |
| - name: Resolve PR base branch | |
| if: github.event_name == 'issue_comment' || github.event_name == 'pull_request' | |
| id: pr_ctx | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| if [ "${{ github.event_name }}" = "pull_request" ]; then | |
| echo "base_ref=${{ github.base_ref }}" >> "$GITHUB_OUTPUT" | |
| else | |
| BASE=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.issue.number }} --jq .base.ref) | |
| echo "base_ref=$BASE" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Download current test results | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: test-results-${{ github.sha }}-${{ github.run_id }} | |
| path: current-results/ | |
| # PR mode (opened or comment-triggered): fetch latest artifact from | |
| # the PR's base branch (e.g. develop or main). | |
| - name: Download baseline results (PR) | |
| if: github.event_name == 'issue_comment' || github.event_name == 'pull_request' | |
| uses: dawidd6/action-download-artifact@v6 | |
| continue-on-error: true | |
| with: | |
| workflow: system-tests.yml | |
| branch: ${{ steps.pr_ctx.outputs.base_ref }} | |
| name_is_regexp: true | |
| name: "test-results-.*" | |
| path: baseline-results/ | |
| if_no_artifact_found: warn | |
| # Manual dispatch with explicit baseline run ID | |
| - name: Download baseline results (manual, explicit run ID) | |
| if: > | |
| github.event_name == 'workflow_dispatch' && | |
| inputs.baseline_run_id != '' | |
| uses: actions/download-artifact@v4 | |
| continue-on-error: true | |
| with: | |
| run-id: ${{ inputs.baseline_run_id }} | |
| name_is_regexp: true | |
| name: "test-results-.*" | |
| path: baseline-results/ | |
| # Manual dispatch without explicit baseline: fetch latest from main | |
| - name: Download baseline results (manual, latest main) | |
| if: > | |
| github.event_name == 'workflow_dispatch' && | |
| inputs.baseline_run_id == '' | |
| uses: dawidd6/action-download-artifact@v6 | |
| continue-on-error: true | |
| with: | |
| workflow: system-tests.yml | |
| branch: main | |
| name_is_regexp: true | |
| name: "test-results-.*" | |
| path: baseline-results/ | |
| if_no_artifact_found: warn | |
| - name: Locate result directories | |
| id: dirs | |
| # Find the dir holding results.xml. Nesting depth differs by downloader: | |
| # actions/download-artifact@v4 (single name) extracts straight into the | |
| # path, while dawidd6/action-download-artifact@v6 with name_is_regexp | |
| # wraps each artifact in a subdir named after it. `find` handles both. | |
| run: | | |
| CURRENT_XML=$(find current-results/ -name results.xml 2>/dev/null | sort -r | head -1) | |
| [ -n "$CURRENT_XML" ] && echo "current=$(dirname "$CURRENT_XML")" >> "$GITHUB_OUTPUT" | |
| BASELINE_XML=$(find baseline-results/ -name results.xml 2>/dev/null | sort -r | head -1) | |
| if [ -n "$BASELINE_XML" ]; then | |
| echo "baseline=$(dirname "$BASELINE_XML")" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "baseline=" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Generate metrics report | |
| id: report | |
| continue-on-error: true | |
| run: | | |
| CURRENT="${{ steps.dirs.outputs.current }}" | |
| BASELINE="${{ steps.dirs.outputs.baseline }}" | |
| if [ -n "$BASELINE" ]; then | |
| python tests/parse_metrics.py \ | |
| --current "$CURRENT" \ | |
| --baseline "$BASELINE" \ | |
| --output report.md | |
| else | |
| python tests/parse_metrics.py \ | |
| --current "$CURRENT" \ | |
| --output report.md | |
| fi | |
| - name: Post PR comment | |
| if: github.event_name == 'issue_comment' || github.event_name == 'pull_request' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| let body; | |
| try { | |
| body = fs.readFileSync('report.md', 'utf8'); | |
| } catch { | |
| body = '_No metrics report generated._'; | |
| } | |
| const header = `## Test Metrics — \`${{ github.sha }}\`\n\n`; | |
| await github.rest.issues.createComment({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: header + body, | |
| }); | |
| - name: Write job summary | |
| if: always() | |
| run: | | |
| if [ -f report.md ]; then | |
| echo "## Test Metrics — \`${{ github.sha }}\`" >> "$GITHUB_STEP_SUMMARY" | |
| echo "" >> "$GITHUB_STEP_SUMMARY" | |
| cat report.md >> "$GITHUB_STEP_SUMMARY" | |
| else | |
| echo "_No metrics report generated._" >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| - name: Fail on regression | |
| if: steps.report.outcome == 'failure' | |
| run: | | |
| echo "::error::Metric regression detected — see the report above for details." | |
| exit 1 |