feat: Add DocVQA task implementation

- Add DocVQA task for document visual question answering
- Use substring-match scorer to handle answer variations
- Update task registry and configuration files
- Support 5,349 validation examples from lmms-lab/DocVQA

DocVQA is an extractive QA task where models extract answers from
document images. Multiple valid answers are provided per question.

🤖 Generated with Claude Code

Co-Authored-By: Claude <[email protected]>

Author: silviase

scripts/make_leaderboard.py

@@ -21,6 +21,9 @@
     "mmmu": "MMMU",
     "cc-ocr": "CC-OCR",
     "cvqa": "CVQA",
+    "ai2d": "AI2D",
+    "blink": "BLINK",
+    "docvqa": "DocVQA",
 }
 
 TASK_CLUSTER_ALIAS = {
@@ -31,11 +34,14 @@
     "VG-VQA": "視覚中心",
     "Heron": "視覚中心",
     "JVB-ItW": "視覚中心",
-    "MulIm-VQA": "非日本語",
-    "MMMU": "非日本語",
-    "LLAVA": "非日本語",
+    "MulIm-VQA": "その他",
+    "MMMU": "英語",
+    "LLAVA": "英語",
     "CC-OCR": "言語・知識中心",
     "CVQA": "視覚中心",
+    "AI2D": "英語",
+    "BLINK": "英語",
+    "DocVQA": "英語",
 }
 
 METRIC_ALIAS = {
@@ -49,6 +55,10 @@
     "mmmu": "Acc",
     "cc-ocr": "macro_f1",
     "substring-match": "Acc",
+    "cvqa": "Acc",
+    "ai2d": "Acc",
+    "blink": "Acc",
+    "docvqa": "Acc",
 }
 
 MODEL_LIST = [

scripts/nvlink/config.sh

@@ -171,9 +171,11 @@ declare -a task_list=(
     "mmmu"
     "llava-bench-in-the-wild"
     "jic-vqa"
+    "cvqa"
     "mecha-ja"
     "ai2d"
     "blink"
+    "docvqa"
 )
 
 # === Metrics Mapping ===
@@ -190,6 +192,8 @@ declare -A METRIC_MAP=(
     ["mecha-ja"]="mecha-ja"
     ["ai2d"]="ai2d"
     ["blink"]="blink"
+    ["cvqa"]="substring-match"
+    ["docvqa"]="substring-match"
 )
 
 # === Function to load .env file ===

src/eval_mm/tasks/__init__.py

@@ -13,6 +13,7 @@
 from .cvqa import CVQA
 from .ai2d import AI2D
 from .blink import BLINK
+from .docvqa import DocVQA
 from .task_registry import TaskRegistry
 from .task import TaskConfig
 
@@ -32,6 +33,7 @@
     "CVQA",
     "AI2D",
     "BLINK",
+    "DocVQA",
     "TaskRegistry",
     "TaskConfig",
 ]

src/eval_mm/tasks/docvqa.py

@@ -0,0 +1,86 @@
+from eval_mm.tasks.task import Task
+from eval_mm.tasks.task_registry import register_task
+from datasets import load_dataset, Dataset
+from PIL import Image
+
+
+@register_task("docvqa", "DocVQA", "doc-vqa")
+class DocVQA(Task):
+    """DocVQA task implementation.
+    
+    DocVQA is a VQA dataset for understanding images of document pages.
+    It uses extractive QA where models need to extract answers from document images.
+    Multiple valid answers are provided for each question.
+    """
+    
+    def __init__(self, config):
+        super().__init__(config)
+    
+    @staticmethod
+    def _prepare_dataset() -> Dataset:
+        """Load DocVQA validation set."""
+        # Load the DocVQA config from lmms-lab/DocVQA dataset
+        ds = load_dataset("lmms-lab/DocVQA", "DocVQA", split="validation")
+        
+        # Rename questionId to question_id for consistency
+        ds = ds.rename_column("questionId", "question_id")
+        
+        return ds
+    
+    @staticmethod
+    def doc_to_text(doc) -> str:
+        """Convert document to text prompt.
+        
+        DocVQA is an extractive QA task, so we just return the question.
+        """
+        return doc['question']
+    
+    @staticmethod
+    def doc_to_visual(doc) -> list[Image.Image]:
+        """Extract image from document."""
+        return [doc['image']]
+    
+    @staticmethod
+    def doc_to_id(doc) -> str:
+        """Return unique question ID."""
+        return str(doc['question_id'])
+    
+    @staticmethod
+    def doc_to_answer(doc) -> list[str]:
+        """Return list of valid answers.
+        
+        DocVQA provides multiple valid answers for each question.
+        We return all of them for evaluation with substring-match scorer.
+        """
+        return doc['answers']
+
+
+def test_docvqa_task():
+    """Test DocVQA task implementation."""
+    from eval_mm.tasks.task import TaskConfig
+    
+    # Create task instance
+    task = DocVQA(TaskConfig(max_dataset_len=10))
+    
+    # Load dataset
+    print("Loading DocVQA dataset...")
+    ds = task.dataset
+    print(f"Dataset size: {len(ds)}")
+    
+    # Test with first example
+    example = ds[0]
+    print(f"\nFirst example:")
+    print(f"  ID: {task.doc_to_id(example)}")
+    print(f"  Question: {task.doc_to_text(example)}")
+    print(f"  Image: {task.doc_to_visual(example)[0]}")
+    print(f"  Valid answers: {task.doc_to_answer(example)}")
+    
+    # Verify data types
+    assert isinstance(task.doc_to_text(example), str)
+    assert isinstance(task.doc_to_visual(example), list)
+    assert all(isinstance(img, Image.Image) for img in task.doc_to_visual(example))
+    assert isinstance(task.doc_to_id(example), str)
+    assert isinstance(task.doc_to_answer(example), list)
+    assert all(isinstance(ans, str) for ans in task.doc_to_answer(example))
+    
+    print("\nAll tests passed!")