feat(ci): xdist backend test workflow

Enable pytest-xdist in the backend CI workflow with per-worker Snuba
isolation. Key changes:

- Static shard fast-path (skip pytest --collect-only on non-PR runs)
- Per-worker Snuba bootstrap step (conditional on XDIST_PER_WORKER_SNUBA)
- Inline pytest invocation with -n 3 --dist=loadfile
- Snuba container log dump on failure for debugging

Author: mchen-sentry

.github/workflows/backend.yml

@@ -3,9 +3,8 @@ name: backend
 on:
   push:
     branches:
-      - master
-  pull_request:
-    types: [opened, synchronize, reopened, labeled]
+      - mchen/tiered-xdist-v2
+      - mchen/tiered-xdist-v2-clean
   workflow_dispatch:
 
 # Cancel in progress workflows on pull_requests.
@@ -204,15 +203,24 @@ jobs:
     needs: [files-changed, prepare-selective-tests]
     name: calculate test shards
     runs-on: ubuntu-24.04
-    timeout-minutes: 5
+    timeout-minutes: ${{ needs.prepare-selective-tests.outputs.has-selected-tests == 'true' && 5 || 1 }}
     outputs:
-      shard-count: ${{ steps.calculate-shards.outputs.shard-count }}
-      shard-indices: ${{ steps.calculate-shards.outputs.shard-indices }}
+      shard-count: ${{ steps.static-shards.outputs.shard-count || steps.calculate-shards.outputs.shard-count }}
+      shard-indices: ${{ steps.static-shards.outputs.shard-indices || steps.calculate-shards.outputs.shard-indices }}
 
     steps:
+      - name: Use default shards (no selective testing)
+        id: static-shards
+        if: needs.prepare-selective-tests.outputs.has-selected-tests != 'true'
+        run: |
+          echo "shard-count=22" >> "$GITHUB_OUTPUT"
+          echo "shard-indices=[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21]" >> "$GITHUB_OUTPUT"
+
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        if: needs.prepare-selective-tests.outputs.has-selected-tests == 'true'
 
       - name: Setup sentry env
+        if: needs.prepare-selective-tests.outputs.has-selected-tests == 'true'
         uses: ./.github/actions/setup-sentry
         id: setup
         with:
@@ -228,8 +236,9 @@ jobs:
 
       - name: Calculate test shards
         id: calculate-shards
+        if: needs.prepare-selective-tests.outputs.has-selected-tests == 'true'
         env:
-          SELECTED_TESTS_FILE: ${{ needs.prepare-selective-tests.outputs.has-selected-tests == 'true' && '.artifacts/selected-tests.txt' || '' }}
+          SELECTED_TESTS_FILE: '.artifacts/selected-tests.txt'
           SELECTED_TEST_COUNT: ${{ needs.prepare-selective-tests.outputs.test-count || '' }}
         run: |
           python3 .github/workflows/scripts/calculate-backend-test-shards.py
@@ -258,9 +267,11 @@ jobs:
         instance: ${{ fromJSON(needs.calculate-shards.outputs.shard-indices) }}
 
     env:
-      # Dynamic total from calculate-shards
       MATRIX_INSTANCE_TOTAL: ${{ needs.calculate-shards.outputs.shard-count }}
       TEST_GROUP_STRATEGY: roundrobin
+      PYTHONHASHSEED: '0'
+      XDIST_PER_WORKER_SNUBA: '1'
+      XDIST_WORKERS: '3'
 
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
@@ -278,6 +289,50 @@ jobs:
           sudo install -m 755 odiff-linux-x64 /usr/local/bin/odiff
           rm odiff-linux-x64
 
+      - name: Bootstrap per-worker Snuba instances
+        if: env.XDIST_PER_WORKER_SNUBA == '1'
+        run: |
+          set -eo pipefail
+          XDIST_N=${XDIST_WORKERS:-3}
+          SNUBA_IMAGE=$(docker inspect snuba-snuba-1 --format '{{.Config.Image}}')
+          SNUBA_NETWORK=$(docker inspect snuba-snuba-1 --format '{{range $k, $v := .NetworkSettings.Networks}}{{$k}}{{end}}')
+          if [ -z "$SNUBA_IMAGE" ] || [ -z "$SNUBA_NETWORK" ]; then
+            echo "ERROR: Could not inspect snuba-snuba-1 container. Is devservices running?"
+            exit 1
+          fi
+
+          for i in $(seq 0 $((XDIST_N - 1))); do
+            (
+              WORKER_DB="default_gw${i}"
+              WORKER_PORT=$((1230 + i))
+              curl -sf 'http://localhost:8123/' --data-binary "CREATE DATABASE IF NOT EXISTS ${WORKER_DB}"
+              docker run --rm --network "$SNUBA_NETWORK" \
+                -e "CLICKHOUSE_DATABASE=${WORKER_DB}" -e "CLICKHOUSE_HOST=clickhouse" \
+                -e "CLICKHOUSE_PORT=9000" -e "CLICKHOUSE_HTTP_PORT=8123" \
+                -e "DEFAULT_BROKERS=kafka:9093" -e "REDIS_HOST=redis" \
+                -e "REDIS_PORT=6379" -e "REDIS_DB=1" -e "SNUBA_SETTINGS=docker" \
+                "$SNUBA_IMAGE" bootstrap --force 2>&1 | tail -3
+              docker run -d --name "snuba-gw${i}" --network "$SNUBA_NETWORK" \
+                -p "${WORKER_PORT}:1218" \
+                -e "CLICKHOUSE_DATABASE=${WORKER_DB}" -e "CLICKHOUSE_HOST=clickhouse" \
+                -e "CLICKHOUSE_PORT=9000" -e "CLICKHOUSE_HTTP_PORT=8123" \
+                -e "DEFAULT_BROKERS=kafka:9093" -e "REDIS_HOST=redis" \
+                -e "REDIS_PORT=6379" -e "REDIS_DB=1" -e "SNUBA_SETTINGS=docker" \
+                -e "DEBUG=1" "$SNUBA_IMAGE" api
+            ) &
+          done
+          wait
+
+          for i in $(seq 0 $((XDIST_N - 1))); do
+            WORKER_PORT=$((1230 + i))
+            for attempt in $(seq 1 30); do
+              if curl -sf "http://localhost:${WORKER_PORT}/health" > /dev/null 2>&1; then
+                echo "[snuba-gw${i}] ready on port ${WORKER_PORT}"; break; fi
+              [ "$attempt" -eq 30 ] && echo "[snuba-gw${i}] FAILED" && docker logs "snuba-gw${i}" 2>&1 | tail -20 && exit 1
+              sleep 1
+            done
+          done
+
       - name: Download selected tests artifact
         if: needs.prepare-selective-tests.outputs.has-selected-tests == 'true'
         uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
@@ -289,7 +344,19 @@ jobs:
         env:
           SELECTED_TESTS_FILE: ${{ needs.prepare-selective-tests.outputs.has-selected-tests == 'true' && '.artifacts/selected-tests.txt' || '' }}
         run: |
-          make test-python-ci
+          python3 -b -m pytest tests \
+            --reuse-db \
+            -n ${XDIST_WORKERS:-3} \
+            --dist=loadfile \
+            --ignore tests/acceptance \
+            --ignore tests/apidocs \
+            --ignore tests/js \
+            --ignore tests/tools \
+            --json-report \
+            --json-report-file=".artifacts/pytest.json" \
+            --json-report-omit=log \
+            --junit-xml=.artifacts/pytest.junit.xml \
+            -o junit_suite_name=pytest
 
       - name: Report failures
         if: ${{ !cancelled() && github.event_name == 'pull_request' }}
@@ -306,6 +373,9 @@ jobs:
       - name: Inspect failure
         if: failure()
         run: |
+          for name in $(docker ps -a --filter "name=snuba-gw" --format '{{.Names}}'); do
+            echo "--- $name ---"; docker logs "$name" 2>&1 | tail -30
+          done
           if command -v devservices; then
             devservices logs
           fi