33-gpu-celery.yaml

# GPU Celery worker (queue "GPU") — the ONLY GPU-using workload. Pinned to the GPU node.
# ENTRYPOINT /opt/entrypoint (waits pg+redis) then /run/start-celery. Node "celery@GPU".
# Requires: GPU node labeled gpu=true and tainted nvidia.com/gpu=present:NoSchedule,
# plus the NVIDIA device plugin (60-nvidia-device-plugin.yaml) advertising nvidia.com/gpu.
apiVersion: apps/v1
kind: Deployment
metadata:
  name: gpu-celery
  namespace: rodan
spec:
  replicas: 1
  selector:
    matchLabels:
      app: gpu-celery
  template:
    metadata:
      labels:
        app: gpu-celery
    spec:
      imagePullSecrets:
        - name: ghcr-pull-secret
      # MIG-backed vGPU workaround: the device plugin (migStrategy=none, the only mode that
      # works in a vGPU guest) hands the container the PARENT GPU, but CUDA needs the MIG
      # compute instance — whose access node /dev/nvidia-caps/nvidia-cap4 the plugin doesn't
      # inject, so cuInit -> CUDA_ERROR_NO_DEVICE. Instead we pin to a specific node and let
      # the NVIDIA runtime inject the MIG instance directly via NVIDIA_VISIBLE_DEVICES (which
      # DOES inject cap4). This means NOT requesting nvidia.com/gpu (that would make the plugin
      # overwrite NVIDIA_VISIBLE_DEVICES with the parent). We select the MIG via "all" rather
      # than a specific MIG UUID: the node has exactly one 1g.24gb MIG instance and this pod is
      # the only GPU workload pinned here, so "all" resolves to that MIG — and, unlike a
      # hardcoded UUID, survives node reboots (which regenerate MIG UUIDs and otherwise CrashLoop
      # the pod with "failed to get device handle from UUID: Not Found").
      # Long-term fix: a non-MIG (time-sliced) vGPU profile removes all of this.
      nodeSelector:
        gpu: "true"
        kubernetes.io/hostname: k3s-gpu-node-2
      tolerations:
        - key: nvidia.com/gpu
          operator: Exists
          effect: NoSchedule
      # k3s exposes the NVIDIA runtime via this RuntimeClass (confirmed present in-cluster).
      runtimeClassName: nvidia
      containers:
        - name: gpu-celery
          image: ghcr.io/ddmal/rodan-gpu-celery:nightly
          args: ["/run/start-celery"]
          envFrom:
            - configMapRef: { name: rodan-config }
            - secretRef: { name: rodan-secrets }
          env:
            - name: CELERY_JOB_QUEUE
              value: "GPU"
            # "all" -> the single 1g.24gb MIG instance on the pinned node (the runtime injects
            # it incl. /dev/nvidia-caps/nvidia-cap4 so CUDA can open it). Reboot-proof: avoids a
            # node-specific MIG UUID that changes on every reboot/MIG-reconfig. See the workaround
            # note above for why we don't request nvidia.com/gpu.
            - name: NVIDIA_VISIBLE_DEVICES
              value: "all"
            - name: NVIDIA_DRIVER_CAPABILITIES
              value: "compute,utility"
            # TF 2.15.1 has no sm_90 (Hopper) cubins, so kernels JIT-compile from PTX on first use
            # (gpu_device.cc warns "could take 30 minutes"). Persist that JIT cache on the resources
            # PVC (instead of the ephemeral /root/.nv) and raise its cap to the 4 GiB max so large
            # fused kernels don't evict — the compile then happens once, not on every pod restart.
            - name: CUDA_CACHE_PATH
              value: "/rodan/data/.cuda-cache"
            - name: CUDA_CACHE_MAXSIZE
              value: "4294967296"   # 4 GiB (CUDA hard max); default is only ~1 GiB
          volumeMounts:
            - name: resources
              mountPath: /rodan/data
          startupProbe:
            exec:
              command: ["celery", "-A", "rodan", "--workdir", "/code/Rodan", "inspect", "ping", "-d", "celery@GPU", "-t", "30"]
            periodSeconds: 45
            timeoutSeconds: 40        # default probe timeout is 1s; celery inspect ping needs far longer
            failureThreshold: 30
          livenessProbe:
            exec:
              command: ["celery", "-A", "rodan", "--workdir", "/code/Rodan", "inspect", "ping", "-d", "celery@GPU", "-t", "30"]
            periodSeconds: 60
            timeoutSeconds: 45
            failureThreshold: 3
          # NOTE: no `nvidia.com/gpu` request here on purpose — the GPU is selected via
          # NVIDIA_VISIBLE_DEVICES above. Requesting it would let the device plugin overwrite
          # that env with the parent GPU UUID and re-break CUDA. The hostname pin + taint
          # toleration keep this the only GPU workload on the node.
          resources:
            requests:
              cpu: "1"
              memory: "4Gi"
            limits:
              cpu: "2"
              memory: "16Gi"
      volumes:
        - name: resources
          persistentVolumeClaim:
            claimName: rodan-resources