feat: Update model list and HPC configurations for nvlink

- Sync MODEL_LIST in make_leaderboard.py with eval.sh
- Add new models: Qwen2-VL variants, Llama-3.2-90B, gemma-3 pt variants, Heron-NVILA-Lite models
- Comment out deprecated models to match eval.sh configuration
- Update eval.sh time limit to 24 hours and use /data directory for caching
- Fix ring.sh to use correct project paths

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <[email protected]>

Author: silviase

scripts/make_leaderboard.py

@@ -52,31 +52,42 @@
 }
 
 MODEL_LIST = [
-    "stabilityai/japanese-instructblip-alpha",
+    # "stabilityai/japanese-instructblip-alpha",
     "stabilityai/japanese-stable-vlm",
-    "SakanaAI/Llama-3-EvoVLM-JP-v2",
     "cyberagent/llava-calm2-siglip",
-    "llm-jp/llm-jp-3-vila-14b",
-    "sbintuitions/sarashina2-vision-8b",
-    "sbintuitions/sarashina2-vision-14b",
-    "MIL-UT/Asagi-14B",
     "llava-hf/llava-1.5-7b-hf",
     "llava-hf/llava-v1.6-mistral-7b-hf",
     "neulab/Pangea-7B-hf",
-    "mistralai/Pixtral-12B-2409",
     "meta-llama/Llama-3.2-11B-Vision-Instruct",
-    "Efficient-Large-Model/VILA1.5-13b",
-    "OpenGVLab/InternVL2-8B",
-    "OpenGVLab/InternVL2-26B",
+    "meta-llama/Llama-3.2-90B-Vision-Instruct",
+    # "OpenGVLab/InternVL2-8B",
+    # "OpenGVLab/InternVL2-26B",
+    "Qwen/Qwen2-VL-7B-Instruct",
+    "Qwen/Qwen2-VL-72B-Instruct",
+    "Qwen/Qwen2.5-VL-3B-Instruct",
     "Qwen/Qwen2.5-VL-7B-Instruct",
     "Qwen/Qwen2.5-VL-32B-Instruct",
     "Qwen/Qwen2.5-VL-72B-Instruct",
+    "gpt-4o-2024-11-20",
+    # "mistralai/Pixtral-12B-2409",
+    "llm-jp/llm-jp-3-vila-14b",
+    # "Efficient-Large-Model/VILA1.5-13b",
+    "SakanaAI/Llama-3-EvoVLM-JP-v2",
     "google/gemma-3-4b-it",
     "google/gemma-3-12b-it",
     "google/gemma-3-27b-it",
+    "google/gemma-3-4b-pt",
+    "google/gemma-3-12b-pt",
+    "google/gemma-3-27b-pt",
+    # "tokyotech-llm/gemma3_4b_exp8-checkpoint-50000",
+    "sbintuitions/sarashina2-vision-8b",
+    "sbintuitions/sarashina2-vision-14b",
     "microsoft/Phi-4-multimodal-instruct",
-    "gpt-4o-2024-11-20",
+    "MIL-UT/Asagi-14B",
+    "turing-motors/Heron-NVILA-Lite-1B",
+    "turing-motors/Heron-NVILA-Lite-2B",
     "turing-motors/Heron-NVILA-Lite-15B",
+    "turing-motors/Heron-NVILA-Lite-33B",
 ]
 
 

scripts/nvlink/eval.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #SBATCH --job-name=llm-jp-eval-mm
-#SBATCH --time=2:00:00
+#SBATCH --time=24:00:00
 #SBATCH --gres=gpu:NUM_GPUS
 #SBATCH --ntasks=1
 #SBATCH --cpus-per-task=8
@@ -20,10 +20,11 @@ num_gpus=${4:-1}  # Default to 1 GPU if not specified
 # PATH config
 export PATH="$HOME/.local/bin:$PATH"
 export ROOT_DIR="/home/silviase/"
-export HF_HOME="$ROOT_DIR/.hf_cache"
-export HF_DATASETS_CACHE=$HF_HOME/datasets
-export HF_HUB_CACHE=$HF_HOME/models
-export APPTAINER_CACHEDIR="$ROOT_DIR/apptainer_cache"
+export DATA_DIR="/data/silviase/"
+export HF_HOME="$DATA_DIR/.hf_cache"
+export HF_DATASETS_CACHE=$DATA_DIR/datasets
+export HF_HUB_CACHE=$DATA_DIR/models
+export APPTAINER_CACHEDIR="$DATA_DIR/apptainer_cache"
 
 # Environment Variables
 export TORCH_COMPILE_DISABLE=1

scripts/nvlink/ring.sh

@@ -7,8 +7,8 @@ EXIT_CODE=${1:-0}
 JOB_NAME=${2:-"Unknown Job"}
 
 # Load environment variables from .env file
-if [ -f "/home/silviase/augeo/.env" ]; then
-    export $(grep -v '^#' /home/silviase/augeo/.env | xargs)
+if [ -f "/home/silviase/llm-jp-eval-mm/.env" ]; then
+    export $(grep -v '^#' /home/silviase/llm-jp-eval-mm/.env | xargs)
 fi
 
 # Check if webhook URL is set
@@ -45,7 +45,7 @@ curl -X POST "$SLACK_WEBHOOK_URL" \
             {"title": "Node", "value": "$NODE", "short": true},
             {"title": "Completed", "value": "$CURRENT_TIME", "short": true}
         ],
-        "footer": "Augeo Project",
+        "footer": "LLM-jp EvalMM Project",
         "ts": $(date +%s)
     }]
 }