MI300X vLLM PD: persistent JIT cache + root-safe stale-lock cleanup, M4 Docker/runtime parity

- job.slurm: per-image persistent JIT cache (AITER/Triton/TorchInductor/MoRI/vLLM)
  with sudo stale-lock cleanup (cache is root-owned by the container, so the
  unprivileged srun user needs sudo to remove dead-owner locks; mirrors AMDSOW
  Milestone4 cleanup_persistent_jit_locks_node). Fixes cross-run AITER baton
  deadlock. Adds SYS_ADMIN/IPC_LOCK caps, nofile=1048576, shm-size 192G,
  /dev/shm mount, and AITER_JIT_DIR/TRITON/TORCHINDUCTOR/MORI/VLLM_CACHE envs.
- server_vllm.sh: M4-sized startup timeout (7200s) for cold AITER compile +
  FULL cudagraph capture; MoRIIO kv-transfer-config (read_mode, allow_full_cudagraph).
- amd-master.yaml/models_vllm.yaml: DSR1-0528 FP8 MI300X disagg matrix + M4 flags.
- launch_mi300x-amds.sh: MI300X multinode branch + cluster-specific injection.

Validated end-to-end: 1k1k TP8 1P1D MTP3 conc=9 -> 90/90 completed,
output_throughput 725 tok/s, mean TPOT 11.6ms, MoRIIO KV transfer over Thor RDMA.

Author: edwinlim0919

.github/configs/amd-master.yaml

@@ -191,7 +191,6 @@ dsr1-fp8-mi300x-vllm-disagg:
           dp-attn: false
           additional-settings:
           - "PREFILL_NODES=1"
-          - "VLLM_MORIIO_CONNECTOR_READ_MODE=1"
         decode:
           num-worker: 1
           tp: 8
@@ -210,7 +209,6 @@ dsr1-fp8-mi300x-vllm-disagg:
           dp-attn: false
           additional-settings:
           - "PREFILL_NODES=1"
-          - "VLLM_MORIIO_CONNECTOR_READ_MODE=1"
         decode:
           num-worker: 1
           tp: 8
@@ -230,7 +228,6 @@ dsr1-fp8-mi300x-vllm-disagg:
           additional-settings:
           - "PREFILL_NODES=1"
           - "PREFILL_DP8EP=true"
-          - "VLLM_MORIIO_CONNECTOR_READ_MODE=1"
         decode:
           num-worker: 1
           tp: 8
@@ -254,7 +251,6 @@ dsr1-fp8-mi300x-vllm-disagg:
           dp-attn: false
           additional-settings:
           - "PREFILL_NODES=1"
-          - "VLLM_MORIIO_CONNECTOR_READ_MODE=1"
         decode:
           num-worker: 3
           tp: 8
@@ -273,7 +269,6 @@ dsr1-fp8-mi300x-vllm-disagg:
           dp-attn: false
           additional-settings:
           - "PREFILL_NODES=1"
-          - "VLLM_MORIIO_CONNECTOR_READ_MODE=1"
         decode:
           num-worker: 2
           tp: 8
@@ -292,7 +287,6 @@ dsr1-fp8-mi300x-vllm-disagg:
           dp-attn: false
           additional-settings:
           - "PREFILL_NODES=2"
-          - "VLLM_MORIIO_CONNECTOR_READ_MODE=1"
         decode:
           num-worker: 1
           tp: 8
@@ -311,7 +305,6 @@ dsr1-fp8-mi300x-vllm-disagg:
           dp-attn: false
           additional-settings:
           - "PREFILL_NODES=2"
-          - "VLLM_MORIIO_CONNECTOR_READ_MODE=1"
         decode:
           num-worker: 2
           tp: 8
@@ -330,7 +323,6 @@ dsr1-fp8-mi300x-vllm-disagg:
           dp-attn: false
           additional-settings:
           - "PREFILL_NODES=2"
-          - "VLLM_MORIIO_CONNECTOR_READ_MODE=1"
         decode:
           num-worker: 1
           tp: 8
@@ -350,7 +342,6 @@ dsr1-fp8-mi300x-vllm-disagg:
           dp-attn: false
           additional-settings:
           - "PREFILL_NODES=2"
-          - "VLLM_MORIIO_CONNECTOR_READ_MODE=1"
         decode:
           num-worker: 1
           tp: 8

.gitignore

@@ -3,3 +3,9 @@
 experimental/multiturn/vllm_benchmark/results/
 .venv/
 AMDSOW_CLUSTER_NOTES.md
+
+# AMDSOW local regression run artifacts (not for upstream)
+.regression_env.sh
+/benchmark_logs/
+/validation_*.json
+/collect_latest_results.py

benchmarks/multi_node/amd_utils/job.slurm

@@ -314,6 +314,26 @@ export RESULT_FILENAME="${RESULT_FILENAME:-}"
 export SPEC_DECODING="${SPEC_DECODING:-}"
 export IS_MULTINODE="${IS_MULTINODE:-false}"
 
+# AMDSOW Milestone4 vLLM runs use persistent per-image JIT/cache dirs. This is
+# required for AITER/Triton/TorchInductor ABI-consistent runtime artifacts; using
+# image-baked aiter/jit/*.so directly can hit torch/c10 undefined-symbol errors.
+_cache_tag() { echo "$1" | sed 's|[/:]|-|g' | grep -oP '.{0,40}$'; }
+export PERSIST_JIT_CACHE="${PERSIST_JIT_CACHE:-1}"
+export JIT_CLEAN_STALE_LOCKS="${JIT_CLEAN_STALE_LOCKS:-1}"
+export JIT_CACHE_HOST_ROOT="${JIT_CACHE_HOST_ROOT:-/tmp/amdsow-jit-cache}"
+export JIT_CACHE_KEY="${JIT_CACHE_KEY:-$(_cache_tag "${DOCKER_IMAGE_NAME}")}"
+if [[ "$ENGINE" == "vllm-disagg" && "$PERSIST_JIT_CACHE" == "1" ]]; then
+    export AITER_JIT_DIR="${AITER_JIT_DIR:-/jit-cache/aiter}"
+    export TRITON_CACHE_DIR="${TRITON_CACHE_DIR:-/jit-cache/triton}"
+    export TORCHINDUCTOR_CACHE_DIR="${TORCHINDUCTOR_CACHE_DIR:-/jit-cache/torchinductor}"
+    export MORI_JIT_CACHE_DIR="${MORI_JIT_CACHE_DIR:-/jit-cache/mori}"
+    export VLLM_CACHE_ROOT="${VLLM_CACHE_ROOT:-/jit-cache/vllm}"
+    JIT_CACHE_DOCKER_MOUNTS="-v ${JIT_CACHE_HOST_ROOT}/${JIT_CACHE_KEY}/aiter:/jit-cache/aiter -v ${JIT_CACHE_HOST_ROOT}/${JIT_CACHE_KEY}/triton:/jit-cache/triton -v ${JIT_CACHE_HOST_ROOT}/${JIT_CACHE_KEY}/torchinductor:/jit-cache/torchinductor -v ${JIT_CACHE_HOST_ROOT}/${JIT_CACHE_KEY}/mori:/jit-cache/mori -v ${JIT_CACHE_HOST_ROOT}/${JIT_CACHE_KEY}/vllm:/jit-cache/vllm"
+else
+    JIT_CACHE_DOCKER_MOUNTS=""
+fi
+export JIT_CACHE_DOCKER_MOUNTS
+
 SANITIZED_USER=$(echo "$USER_NAME" | tr -c 'a-zA-Z0-9_.-' '_')
 export DOCKER_CONT_NAME="container_${ENGINE}_${SANITIZED_USER}_${MODEL_NAME}_${SLURM_JOB_ID}"
 
@@ -343,6 +363,25 @@ srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c '
     echo "NFS cache refreshed on $(hostname)"
 '
 
+if [[ "$ENGINE" == "vllm-disagg" && "$PERSIST_JIT_CACHE" == "1" ]]; then
+    echo "Preparing persistent JIT caches on selected nodes: ${JIT_CACHE_HOST_ROOT}/${JIT_CACHE_KEY}"
+    srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c '
+        set -euo pipefail
+        JIT_CACHE_DIR="${JIT_CACHE_HOST_ROOT}/${JIT_CACHE_KEY}"
+        mkdir -p "${JIT_CACHE_DIR}/aiter" "${JIT_CACHE_DIR}/triton" "${JIT_CACHE_DIR}/torchinductor" "${JIT_CACHE_DIR}/mori" "${JIT_CACHE_DIR}/vllm"
+        if [[ "${JIT_CLEAN_STALE_LOCKS:-1}" == "1" ]]; then
+            # JIT cache files are created by the root-running container, so the
+            # unprivileged srun user cannot delete stale locks without sudo.
+            # Mirrors AMDSOW Milestone4 cleanup_persistent_jit_locks_node().
+            removed=$(sudo find "${JIT_CACHE_DIR}" -type f \( -name "lock" -o -name ".lock" -o -name "lock_*" -o -name "*.lock" \) -print -delete 2>/dev/null | wc -l || true)
+            if [[ "${removed//[[:space:]]/}" != "0" ]]; then
+                echo "[jit-cache] removed ${removed//[[:space:]]/} stale lock file(s) under ${JIT_CACHE_DIR} on $(hostname)"
+            fi
+        fi
+        echo "[jit-cache] ready ${JIT_CACHE_DIR} on $(hostname)"
+    '
+fi
+
 # =============================================================================
 # Build engine-specific Docker environment variables
 # =============================================================================
@@ -411,6 +450,11 @@ if [[ "$ENGINE" == "vllm-disagg" ]]; then
         -e HSA_ENABLE_SDMA=1
         -e PROXY_STREAM_IDLE_TIMEOUT=\${PROXY_STREAM_IDLE_TIMEOUT:-300}
         -e VLLM_MORIIO_CONNECTOR_READ_MODE=\${VLLM_MORIIO_CONNECTOR_READ_MODE:-1}
+        -e AITER_JIT_DIR=${AITER_JIT_DIR:-}
+        -e TRITON_CACHE_DIR=${TRITON_CACHE_DIR:-}
+        -e TORCHINDUCTOR_CACHE_DIR=${TORCHINDUCTOR_CACHE_DIR:-}
+        -e MORI_JIT_CACHE_DIR=${MORI_JIT_CACHE_DIR:-}
+        -e VLLM_CACHE_ROOT=${VLLM_CACHE_ROOT:-}
         -e PYTHONPYCACHEPREFIX=/tmp/pycache
     )
 elif [[ "$ENGINE" == "atom-disagg" ]]; then
@@ -534,6 +578,7 @@ fi  # end: if ENGINE == atom-disagg
 \$DOCKER_CMD ps -aq --filter \"$CONT_FILTER\" | xargs -r \$DOCKER_CMD rm -f || true
 \$DOCKER_CMD ps -aq | xargs -r \$DOCKER_CMD stop || true
 
+
 # Start vLLM external router container on node 0
 if [[ \"$ENGINE\" == \"vllm-disagg\" && \"$ROUTER_TYPE\" == \"vllm-router\" && \"\$SLURM_PROCID\" == \"0\" ]]; then
     \$DOCKER_CMD rm -f \"$ROUTER_CONT_NAME\" 2>/dev/null || true
@@ -580,21 +625,26 @@ fi
     --device=/dev/infiniband/uverbs7 \
     --ulimit memlock=-1 \
     --ulimit stack=67108864 \
+    --ulimit nofile=1048576:1048576 \
     --network host \
     --ipc host \
     --group-add video \
+    --cap-add SYS_ADMIN \
+    --cap-add IPC_LOCK \
     --cap-add SYS_PTRACE \
     --security-opt seccomp=unconfined \
     --privileged \
+    --shm-size 192G \
     -v /sys:/sys \
+    -v /dev/shm:/dev/shm \
     $(command -v nicctl >/dev/null 2>&1 && echo "-v $(which nicctl):/usr/sbin/nicctl") \
     -v ${MODEL_DIR}:/models \
     -v \$HOME/.ssh:/root/.ssh \
-    --shm-size 128G \
     -v /tmp:/run_logs \
     -v ${BENCHMARK_LOGS_DIR}:/benchmark_logs \
     -v ${DI_REPO_DIR}:${DOCKER_MOUNT_PATH} \
     ${EXTRA_DOCKER_MOUNTS:-} \
+    ${JIT_CACHE_DOCKER_MOUNTS:-} \
     \${RDMA_MOUNTS[@]+"\${RDMA_MOUNTS[@]}"} \
     ${DOCKER_ENV_COMMON[*]} \
     ${DOCKER_ENV_ENGINE[*]} \

benchmarks/multi_node/amd_utils/models_vllm.yaml

@@ -46,8 +46,23 @@ gpt-oss-120b:
 DeepSeek-R1-0528:
   # DSR1-0528 FP8 MI300X vLLM PD-disaggregation (AMDSOW Milestone4).
   # Tensor-parallel size is appended by server_vllm.sh from PREFILL_TP_SIZE / DECODE_TP_SIZE.
-  # MTP speculative-config is injected by server_vllm.sh from DECODE_MTP_SIZE (decode only);
+  # MTP speculative-config is injected by server_vllm.sh from DECODE_MTP_SIZE onto BOTH
+  # producer+consumer (deepseek_mtp needs matching MTP layers across the PD pair);
   # do NOT hardcode --speculative-config here or the per-row MTP sizes won't take effect.
+  #
+  # The flags/env below are the TP8 profile (AMDSOW exp_common.sh non-DP8EP family):
+  # block-size 64, FULL_AND_PIECEWISE + pass_config (SP/fuse_rope), Quick Reduce, and
+  # fusion-shared-experts=1. DP8EP wide-EP rows use the separate dp8ep_* profile below.
   prefill_flags: "--dtype auto --trust-remote-code --distributed-executor-backend mp --kv-cache-dtype fp8_e4m3 --block-size 64 --performance-mode throughput --no-enable-chunked-prefill --compilation-config '{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"pass_config\":{\"fuse_rope_kvcache\":true,\"fuse_rope_kvcache_cat_mla\":true,\"enable_sp\":true,\"fuse_gemm_comms\":true,\"fuse_act_padding\":true}}' --no-enable-prefix-caching --max-model-len 32768 --max-num-batched-tokens 32768 --gpu-memory-utilization 0.80"
   decode_flags: "--dtype auto --trust-remote-code --distributed-executor-backend mp --kv-cache-dtype fp8_e4m3 --block-size 64 --performance-mode throughput --no-enable-chunked-prefill --compilation-config '{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"pass_config\":{\"fuse_rope_kvcache\":true,\"fuse_rope_kvcache_cat_mla\":true,\"enable_sp\":true,\"fuse_gemm_comms\":true,\"fuse_act_padding\":true}}' --no-enable-prefix-caching --max-model-len 32768 --max-num-batched-tokens 32768 --gpu-memory-utilization 0.80"
-  env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_LINEAR=True VLLM_ROCM_USE_AITER_MLA=True VLLM_ROCM_USE_AITER_MOE=True VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16=1 VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB=2048 VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=3600 VLLM_MORIIO_CONNECTOR_READ_MODE=true VLLM_MOE_DP_CHUNK_SIZE=1024 AITER_ENABLE_VSKIP=0 AITER_ONLINE_TUNE=0 AITER_BYPASS_TUNE_CONFIG=1 MORI_EP_LAUNCH_CONFIG_MODE=AUTO MORI_SHMEM_HEAP_SIZE=24G MORI_SHMEM_MODE=ISOLATION MORI_APP_LOG_LEVEL=INFO MORI_IO_QP_MAX_SEND_WR=8192 MORI_IO_QP_MAX_CQE=16384 MORI_IO_QP_MAX_SGE=4 MORI_RDMA_TC=104 MORI_IB_PATH_MTU=1024 NCCL_IB_DISABLE=0 MORI_ALLOW_FULL_CUDAGRAPH=1"
+  env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_LINEAR=True VLLM_ROCM_USE_AITER_MLA=True VLLM_ROCM_USE_AITER_MOE=True VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16=1 VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB=2048 VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=3600 VLLM_MORIIO_CONNECTOR_READ_MODE=true VLLM_MOE_DP_CHUNK_SIZE=1024 AITER_ENABLE_VSKIP=0 AITER_ONLINE_TUNE=0 AITER_BYPASS_TUNE_CONFIG=1 MORI_EP_LAUNCH_CONFIG_MODE=AUTO MORI_SHMEM_HEAP_SIZE=24G MORI_SHMEM_MODE=ISOLATION MORI_APP_LOG_LEVEL=INFO MORI_IO_QP_MAX_SEND_WR=16384 MORI_IO_QP_MAX_CQE=32768 MORI_IO_QP_MAX_SGE=4 MORI_RDMA_TC=104 MORI_IB_PATH_MTU=1024 NCCL_IB_DISABLE=0 MORI_ALLOW_FULL_CUDAGRAPH=1"
+  # DP8EP (wide-EP) profile — selected per-role by server_vllm.sh when PREFILL_DP8EP /
+  # DECODE_DP8EP is true. Mirrors AMDSOW exp_common.sh dp8ep family: block-size 1,
+  # cudagraph_mode-only compilation (NO pass_config), no --no-enable-chunked-prefill,
+  # max-model-len 10240, max-num-batched-tokens 8192, --num-gpu-blocks-override 1372000,
+  # fusion-shared-experts=0, and Quick Reduce OFF. The parallelism flags
+  # (--tensor-parallel-size 1 --data-parallel-size <gpus> --enable-expert-parallel
+  # --all2all-backend mori) are appended by server_vllm.sh from GPUS_PER_NODE.
+  dp8ep_prefill_flags: "--dtype auto --trust-remote-code --distributed-executor-backend mp --kv-cache-dtype fp8_e4m3 --block-size 1 --performance-mode throughput --compilation-config '{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}' --no-enable-prefix-caching --max-model-len 10240 --max-num-batched-tokens 8192 --gpu-memory-utilization 0.80 --num-gpu-blocks-override 1372000"
+  dp8ep_decode_flags: "--dtype auto --trust-remote-code --distributed-executor-backend mp --kv-cache-dtype fp8_e4m3 --block-size 1 --performance-mode throughput --compilation-config '{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}' --no-enable-prefix-caching --max-model-len 10240 --max-num-batched-tokens 8192 --gpu-memory-utilization 0.80 --num-gpu-blocks-override 1372000"
+  dp8ep_env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_LINEAR=True VLLM_ROCM_USE_AITER_MLA=True VLLM_ROCM_USE_AITER_MOE=True VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0 VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=3600 VLLM_MORIIO_CONNECTOR_READ_MODE=true VLLM_MOE_DP_CHUNK_SIZE=1024 AITER_ENABLE_VSKIP=0 AITER_ONLINE_TUNE=0 AITER_BYPASS_TUNE_CONFIG=1 MORI_EP_LAUNCH_CONFIG_MODE=AUTO MORI_SHMEM_HEAP_SIZE=24G MORI_SHMEM_MODE=ISOLATION MORI_APP_LOG_LEVEL=INFO MORI_IO_QP_MAX_SEND_WR=16384 MORI_IO_QP_MAX_CQE=32768 MORI_IO_QP_MAX_SGE=4 MORI_RDMA_TC=104 MORI_IB_PATH_MTU=1024 NCCL_IB_DISABLE=0 MORI_ALLOW_FULL_CUDAGRAPH=1"