Minor manifests fixes

Makefile

@@ -19,7 +19,7 @@ build:
 	@docker compose -f build.yml build --no-cache py3-celery
 	# rodan(-main) and rodan-client next — nginx is FROM rodan-main.
 	@docker compose -f build.yml build --no-cache --parallel rodan rodan-client
-	@docker compose -f build.yml build --no-cache --parallel nginx gpu-celery postgres
+	@docker compose -f build.yml build --no-cache --parallel nginx gpu-celery postgres iipsrv
 	@echo "[+] Done."
 
 run: remote_jobs
@@ -50,14 +50,21 @@ clean_git:
 health:
 	@docker inspect --format "{{json .State.Health }}" $(log) | jq
 
-$(JOBS_PATH)/neon_wrapper/Neon/package.json:
-	@cd $(JOBS_PATH); \
-		git clone --recurse-submodules -b develop https://github.com/DDMAL/neon_wrapper.git
-
-$(JOBS_PATH)/neon_wrapper/static/editor.html: $(JOBS_PATH)/neon_wrapper/Neon/package.json
-	@cd $(JOBS_PATH)/neon_wrapper; \
-		yarn install && \
-		yarn build
+# Neon's webpack build needs the exact toolchain baked into the rodan-python3-celery image
+# (Alpine node 16 / yarn) and fails on newer host Node versions. So instead of building on the
+# host, extract the already-built neon_wrapper (source + static/editor.html + Python package)
+# out of that image — built by `make build`, or pulled from GHCR if absent.
+$(JOBS_PATH)/neon_wrapper/static/editor.html:
+	@echo "[-] Extracting pre-built neon_wrapper from rodan-python3-celery:$(DOCKER_TAG)..."
+	@docker image inspect ghcr.io/ddmal/rodan-python3-celery:$(DOCKER_TAG) >/dev/null 2>&1 || \
+		docker pull ghcr.io/ddmal/rodan-python3-celery:$(DOCKER_TAG)
+	@# Remove any existing copy as root — the containers run as root and may own __pycache__
+	@# files under the bind-mounted jobs dir that the host user cannot delete.
+	@docker run --rm --entrypoint sh -v $(abspath $(JOBS_PATH)):/jobs ghcr.io/ddmal/rodan-python3-celery:$(DOCKER_TAG) -c "rm -rf /jobs/neon_wrapper"
+	@cid=$$(docker create ghcr.io/ddmal/rodan-python3-celery:$(DOCKER_TAG)); \
+		docker cp $$cid:/code/Rodan/rodan/jobs/neon_wrapper $(JOBS_PATH)/neon_wrapper; \
+		docker rm $$cid >/dev/null
+	@echo "[+] neon_wrapper ready."
 
 $(JOBS_PATH)/pixel_wrapper/package.json:
 	@cd $(JOBS_PATH); git clone --recurse-submodules -b develop https://github.com/DDMAL/pixel_wrapper.git

build.yml

@@ -7,13 +7,13 @@ services:
       dockerfile: Dockerfile
       args:
         VERSION: nightly
-    image: "ddmal/nginx:nightly"
+    image: "ghcr.io/ddmal/nginx:nightly"
 
   iipsrv:
     build:
       context: ./iipsrv
       dockerfile: Dockerfile
-    image: "ddmal/iipsrv:nightly"
+    image: "ghcr.io/ddmal/iipsrv:nightly"
 
   rodan:
     build:
@@ -22,32 +22,42 @@ services:
       args:
         BRANCHES: develop
         VERSION: nightly
-    image: "ddmal/rodan-main:nightly"
+      # nginx/Dockerfile does `FROM ddmal/rodan-main:${VERSION}` (Docker Hub namespace).
+      # Emit that tag alongside the ghcr one so a local `make build` resolves the base
+      # without a manual retag (CI does the same via an extra `-t ddmal/rodan-main:$TAG`).
+      tags:
+        - "ddmal/rodan-main:nightly"
+    image: "ghcr.io/ddmal/rodan-main:nightly"
 
   py3-celery:
     build:
       context: .
       dockerfile: ./python3-celery/Dockerfile
       args:
         BRANCHES: develop
-    image: "ddmal/rodan-python3-celery:nightly"
+      # rodan-main/Dockerfile does `FROM ddmal/rodan-python3-celery:${VERSION}` (Docker Hub
+      # namespace). Emit that tag alongside the ghcr one so a local `make build` resolves the
+      # base without a manual retag (CI does the same via an extra `-t ddmal/...:$TAG`).
+      tags:
+        - "ddmal/rodan-python3-celery:nightly"
+    image: "ghcr.io/ddmal/rodan-python3-celery:nightly"
 
   gpu-celery:
     build:
       context: .
       dockerfile: ./gpu-celery/Dockerfile
       args:
         BRANCHES: develop
-    image: "ddmal/rodan-gpu-celery:nightly"
+    image: "ghcr.io/ddmal/rodan-gpu-celery:nightly"
 
   postgres:
     build:
       context: .
       dockerfile: ./postgres/Dockerfile
-    image: "ddmal/postgres-plpython:nightly"
+    image: "ghcr.io/ddmal/postgres-plpython:nightly"
 
   rodan-client:
     build:
       context: ./rodan-client
       dockerfile: Dockerfile
-    image: "ddmal/rodan-client:nightly"
+    image: "ghcr.io/ddmal/rodan-client:nightly"

docker-compose.yml

@@ -1,7 +1,7 @@
 version: "3.4"
 services:
   nginx:
-    image: "ddmal/nginx:${DOCKER_TAG}"
+    image: "ghcr.io/ddmal/nginx:${DOCKER_TAG}"
     command: /run/start
     depends_on:
       - celery
@@ -18,9 +18,13 @@ services:
       - "9002:9002"
     volumes:
       - "resources:/rodan/data"
+      # Local-only override: pass the real request scheme instead of the hardcoded
+      # `X-Scheme https` baked into the prod image, so DRF emits http:// URLs over
+      # plain-HTTP local dev (Django's SECURE_PROXY_SSL_HEADER trusts X-Scheme).
+      - "./nginx/config/rodan.local.conf:/etc/nginx/sites-available/rodan.conf"
 
   rodan-main:
-    image: "ddmal/rodan-main:${DOCKER_TAG}"
+    image: "ghcr.io/ddmal/rodan-main:${DOCKER_TAG}"
     healthcheck:
       test:
         [
@@ -31,7 +35,7 @@ services:
       timeout: "5s"
       retries: 2
       start_period: "2m"
-    command: bash -c "tail -f /dev/null"
+    command: /run/start
     environment:
       TZ: America/Toronto
       SERVER_HOST: localhost
@@ -48,19 +52,19 @@ services:
       - "./rodan-main/code:/code/Rodan"
 
   rodan-client:
-    image: "ddmal/rodan-client:${DOCKER_TAG}"
+    image: "ghcr.io/ddmal/rodan-client:${DOCKER_TAG}"
     volumes:
       - "./rodan-client/code:/code"
       - "./rodan-client/config/configuration.json:/client/configuration.json"
 
   iipsrv:
-    image: "ddmal/iipsrv:${DOCKER_TAG}"
+    image: "ghcr.io/ddmal/iipsrv:${DOCKER_TAG}"
     volumes:
       - "resources:/rodan/data"
 
   celery:
-    image: "ddmal/rodan-main:${DOCKER_TAG}"
-    command: bash -c "tail -f /dev/null"
+    image: "ghcr.io/ddmal/rodan-main:${DOCKER_TAG}"
+    command: /run/start-celery
     environment:
       TZ: America/Toronto
       SERVER_HOST: localhost
@@ -70,12 +74,12 @@ services:
         [
           "CMD",
           "celery",
-          "inspect",
-          "ping",
           "-A",
           "rodan",
           "--workdir",
           "/code/Rodan",
+          "inspect",
+          "ping",
           "-d",
           "celery@celery"
         ]
@@ -95,7 +99,7 @@ services:
       - "./rodan-main/code:/code/Rodan"
 
   py3-celery:
-    image: "ddmal/rodan-python3-celery:${DOCKER_TAG}"
+    image: "ghcr.io/ddmal/rodan-python3-celery:${DOCKER_TAG}"
     command: bash -c "tail -f /dev/null"
     environment:
       TZ: America/Toronto
@@ -114,8 +118,8 @@ services:
       - "./rodan-main/code:/code/Rodan"
 
   gpu-celery:
-    image: "ddmal/rodan-gpu-celery:${DOCKER_TAG}"
-    command: bash -c "tail -f /dev/null"
+    image: "ghcr.io/ddmal/rodan-gpu-celery:${DOCKER_TAG}"
+    command: /run/start-celery
     environment:
       TZ: America/Toronto
       SERVER_HOST: localhost
@@ -143,7 +147,7 @@ services:
       - postgres
 
   postgres:
-    image: "ddmal/postgres-plpython:${DOCKER_TAG}"
+    image: "ghcr.io/ddmal/postgres-plpython:${DOCKER_TAG}"
     healthcheck:
       test: [ "CMD-SHELL", "pg_isready", "-U", "postgres" ]
       interval: 10s
@@ -153,7 +157,11 @@ services:
       - ./scripts/local.env
 
   rabbitmq:
-    image: "rabbitmq:alpine"
+    # Pinned to the 3.13 line to match production (3.13.7). RabbitMQ 4.x disables the
+    # `transient_nonexcl_queues` feature by default, which Celery's pidbox/reply/event
+    # queues (and `celery inspect`) declare — an unpinned `rabbitmq:alpine` now resolves
+    # to 4.x and breaks the workers with "Feature transient_nonexcl_queues is deprecated".
+    image: "rabbitmq:3.13-alpine"
     healthcheck:
       test: [ "CMD", "rabbitmq-diagnostics", "-q", "ping" ]
       interval: "30s"

iipsrv/Dockerfile

@@ -11,6 +11,24 @@ RUN apk add --no-cache \
 
 RUN wget https://github.com/ruven/iipsrv/archive/refs/heads/master.zip && unzip master.zip
 WORKDIR /iipsrv-master
+# Patch upstream OpenJPEG 8-bit decode bug: the mask for 8-bit output data is
+# 0x000000f (4-bit / low nibble only), which renders every 8-bit JP2 ~32x too
+# dark. It must be 0x000000ff (full 8-bit byte). See src/OpenJPEGImage.cc.
+RUN sed -i 's/& 0x000000f)/\& 0x000000ff)/' src/OpenJPEGImage.cc \
+  && grep -q '& 0x000000ff)' src/OpenJPEGImage.cc
+# Patch upstream OpenJPEG region-clamp bug in OpenJPEGImage::process().
+# w0/h0 are END coordinates passed to
+# opj_set_decode_area(codec, img, start_x, start_y, end_x, end_y),
+# but the overrun clamp treats them as widths (x0 + w0) and reassigns
+# w0 = image_widths[0] - x0. For any tile whose x0 > width/2 this makes
+# end_x < start_x, so OpenJPEG gets a negative decode width and errors out
+# -> the right/bottom half of every image 404s (and the boundary band
+# garbles) when zooming. w0/h0 must be clamped to the image dimensions
+# directly, not offset by x0/y0.
+RUN sed -i 's/if( x0 + w0 > image_widths\[0\][[:space:]]*) w0 = image_widths\[0\][[:space:]]*- x0;/if( w0 > image_widths[0] ) w0 = image_widths[0];/' src/OpenJPEGImage.cc \
+  && sed -i 's/if( y0 + h0 > image_heights\[0\][[:space:]]*) h0 = image_heights\[0\][[:space:]]*- y0;/if( h0 > image_heights[0] ) h0 = image_heights[0];/' src/OpenJPEGImage.cc \
+  && grep -q 'if( w0 > image_widths\[0\] ) w0 = image_widths\[0\];' src/OpenJPEGImage.cc \
+  && grep -q 'if( h0 > image_heights\[0\] ) h0 = image_heights\[0\];' src/OpenJPEGImage.cc
 RUN ./autogen.sh
 RUN ./configure --enable-openjpeg
 RUN make

k8s/33-gpu-celery.yaml

@@ -25,8 +25,12 @@ spec:
       # inject, so cuInit -> CUDA_ERROR_NO_DEVICE. Instead we pin to a specific node and let
       # the NVIDIA runtime inject the MIG instance directly via NVIDIA_VISIBLE_DEVICES (which
       # DOES inject cap4). This means NOT requesting nvidia.com/gpu (that would make the plugin
-      # overwrite NVIDIA_VISIBLE_DEVICES with the parent). The MIG UUID is per-node, hence the
-      # hostname pin. Long-term fix: a non-MIG (time-sliced) vGPU profile removes all of this.
+      # overwrite NVIDIA_VISIBLE_DEVICES with the parent). We select the MIG via "all" rather
+      # than a specific MIG UUID: the node has exactly one 1g.24gb MIG instance and this pod is
+      # the only GPU workload pinned here, so "all" resolves to that MIG — and, unlike a
+      # hardcoded UUID, survives node reboots (which regenerate MIG UUIDs and otherwise CrashLoop
+      # the pod with "failed to get device handle from UUID: Not Found").
+      # Long-term fix: a non-MIG (time-sliced) vGPU profile removes all of this.
       nodeSelector:
         gpu: "true"
         kubernetes.io/hostname: k3s-gpu-node-2
@@ -46,11 +50,12 @@ spec:
           env:
             - name: CELERY_JOB_QUEUE
               value: "GPU"
-            # MIG compute-instance UUID on k3s-gpu-node-2 (nvidia-smi -L). The runtime
-            # injects this exact MIG device (incl. /dev/nvidia-caps/nvidia-cap4) so CUDA can
-            # open it. This is node-specific — it must match the node pinned above.
+            # "all" -> the single 1g.24gb MIG instance on the pinned node (the runtime injects
+            # it incl. /dev/nvidia-caps/nvidia-cap4 so CUDA can open it). Reboot-proof: avoids a
+            # node-specific MIG UUID that changes on every reboot/MIG-reconfig. See the workaround
+            # note above for why we don't request nvidia.com/gpu.
             - name: NVIDIA_VISIBLE_DEVICES
-              value: "MIG-a724b14d-5057-529c-b9ef-87ebd0d2d92d"
+              value: "all"
             - name: NVIDIA_DRIVER_CAPABILITIES
               value: "compute,utility"
             # TF 2.15.1 has no sm_90 (Hopper) cubins, so kernels JIT-compile from PTX on first use

nginx/config/rodan.conf

@@ -18,6 +18,7 @@ server { # Rodan API
     }
     proxy_set_header  Host              $http_host;
     fastcgi_param     QUERY_STRING      $query_string;
+    fastcgi_param     REQUEST_URI       $request_uri;
     fastcgi_param     REQUEST_METHOD    $request_method;
     fastcgi_param     CONTENT_TYPE      $content_type;
     fastcgi_param     CONTENT_LENGTH    $content_length;

nginx/config/rodan.local.conf

@@ -0,0 +1,76 @@
+server { # Rodan API
+  listen        80;
+  listen        [::]:80; # IPv6
+  access_log    /var/log/nginx/rodan.log  main;
+  include       /etc/nginx/conf.d/error.conf;
+
+  location /static/  { # Static files for API
+    limit_except GET {
+      deny       all;
+    }
+    root /rodan;
+  }
+
+  location /fcgi-bin/iipsrv.fcgi { # iipsrv
+    fastcgi_pass   iipsrv:9003;
+    limit_except   GET {
+      deny         all;
+    }
+    proxy_set_header  Host              $http_host;
+    fastcgi_param     QUERY_STRING      $query_string;
+    fastcgi_param     REQUEST_URI       $request_uri;
+    fastcgi_param     REQUEST_METHOD    $request_method;
+    fastcgi_param     CONTENT_TYPE      $content_type;
+    fastcgi_param     CONTENT_LENGTH    $content_length;
+    fastcgi_param     PATH_INFO         $fastcgi_script_name;
+  }
+
+  location /uploads { # NFS storage
+    limit_except      GET {
+      deny            all;
+    }
+    add_header      Host $http_host;
+    add_header      'Access-Control-Allow-Origin' '*';
+    proxy_pass_request_headers      on;
+    alias /rodan/data;
+  }
+
+  location /ws/ { # WebSocket broadcast (Django Channels ASGI on rodan-main; replaces ws4redis)
+    proxy_pass            http://rodan-main:8000;
+    proxy_http_version    1.1;
+    proxy_set_header      Upgrade $http_upgrade;
+    proxy_set_header      Connection "upgrade";
+    proxy_set_header      Host $http_host;
+    proxy_set_header      X-Real-IP $remote_addr;
+    proxy_set_header      X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header      X-Forwarded-Proto https;
+    proxy_connect_timeout 10;
+    proxy_read_timeout    3600; # long-lived broadcast socket
+    proxy_send_timeout    3600;
+  }
+
+  location /ht { # Django Health Check
+    limit_except      GET {
+      deny            all;
+    }
+    proxy_pass            http://rodan-main:8000/ht;
+  }
+
+  location /api { # Gunicorn Portal
+    limit_except      GET POST HEAD OPTIONS PATCH PUT DELETE {
+      deny            all;
+    }
+    proxy_set_header      Host $http_host;
+    proxy_set_header      X-Scheme $scheme;
+    proxy_set_header      'Access-Control-Allow-Origin' '*';
+    proxy_set_header      'Access-Control-Allow-Headers' 'Content-Type';
+    proxy_pass_request_headers      on;
+    proxy_pass            http://rodan-main:8000;
+  }
+
+  location / {
+    proxy_pass_request_headers on;
+    include /etc/nginx/conf.d/error.conf;
+    proxy_pass http://rodan-client;
+  }
+}