Adds multimodal support and MMMU pro

examples/model_configs/transformers_vlm_model.yaml

@@ -0,0 +1,10 @@
+model_parameters:
+  model_name: "Qwen/Qwen2.5-VL-3B-Instruct"
+  revision: "main"
+  dtype: "float16"
+  compile: false
+  model_parallel: false
+  batch_size: 1
+  generation_parameters:
+    temperature: 0.2
+    top_p: 0.9

pyproject.toml

@@ -54,7 +54,7 @@ classifiers = [
 keywords = ["evaluation", "nlp", "llm"]
 dependencies = [
     # Base dependencies
-    "transformers>=4.38.0",
+    "transformers>=4.51.0",
     "accelerate",
     "huggingface_hub[hf_xet]>=0.30.2",
     "torch>=2.0,<3.0",

src/lighteval/main_accelerate.py

@@ -48,6 +48,9 @@ def accelerate(  # noqa C901
     use_chat_template: Annotated[
         bool, Option(help="Use chat template for evaluation.", rich_help_panel=HELP_PANEL_NAME_4)
     ] = False,
+    vision_model: Annotated[
+        bool, Option(help="Use vision model for evaluation.", rich_help_panel=HELP_PANEL_NAME_4)
+    ] = False,
     system_prompt: Annotated[
         Optional[str], Option(help="Use system prompt for evaluation.", rich_help_panel=HELP_PANEL_NAME_4)
     ] = None,
@@ -109,6 +112,7 @@ def accelerate(  # noqa C901
     from lighteval.models.transformers.adapter_model import AdapterModelConfig
     from lighteval.models.transformers.delta_model import DeltaModelConfig
     from lighteval.models.transformers.transformers_model import TransformersModelConfig
+    from lighteval.models.transformers.vlm_transformers_model import VLMTransformersModelConfig
     from lighteval.models.utils import ModelConfig
     from lighteval.pipeline import ParallelismManager, Pipeline, PipelineParameters
 
@@ -147,7 +151,10 @@ def accelerate(  # noqa C901
     elif config.get("adapter_weights", False):
         model_config = AdapterModelConfig(**config)
     else:
-        model_config = TransformersModelConfig(**config)
+        if vision_model:
+            model_config = VLMTransformersModelConfig(**config)
+        else:
+            model_config = TransformersModelConfig(**config)
 
     pipeline = Pipeline(
         tasks=tasks,

src/lighteval/models/model_loader.py

@@ -42,6 +42,8 @@
 from lighteval.models.transformers.adapter_model import AdapterModel, AdapterModelConfig
 from lighteval.models.transformers.delta_model import DeltaModel, DeltaModelConfig
 from lighteval.models.transformers.transformers_model import TransformersModel, TransformersModelConfig
+from lighteval.models.transformers.vlm_transformers_model import VLMTransformersModel, VLMTransformersModelConfig
+from lighteval.models.utils import ModelConfig
 from lighteval.models.vllm.vllm_model import VLLMModel, VLLMModelConfig
 from lighteval.utils.imports import (
     NO_LITELLM_ERROR_MSG,
@@ -60,21 +62,8 @@
 
 
 def load_model(  # noqa: C901
-    config: Union[
-        TransformersModelConfig,
-        AdapterModelConfig,
-        DeltaModelConfig,
-        TGIModelConfig,
-        InferenceEndpointModelConfig,
-        DummyModelConfig,
-        VLLMModelConfig,
-        CustomModelConfig,
-        OpenAIModelConfig,
-        LiteLLMModelConfig,
-        SGLangModelConfig,
-        InferenceProvidersModelConfig,
-    ],
-) -> Union[TransformersModel, AdapterModel, DeltaModel, ModelClient, DummyModel]:
+    config: ModelConfig,
+) -> LightevalModel:
     """Will load either a model from an inference server or a model from a checkpoint, depending
     on the config type.
 
@@ -100,6 +89,9 @@ def load_model(  # noqa: C901
     if isinstance(config, TransformersModelConfig):
         return load_model_with_accelerate_or_default(config)
 
+    if isinstance(config, VLMTransformersModelConfig):
+        return load_model_with_accelerate_or_default(config)
+
     if isinstance(config, DummyModelConfig):
         return load_dummy_model(config)
 
@@ -186,7 +178,9 @@ def load_model_with_inference_endpoints(config: Union[InferenceEndpointModelConf
 
 
 def load_model_with_accelerate_or_default(
-    config: Union[AdapterModelConfig, TransformersModelConfig, DeltaModelConfig],
+    config: Union[
+        AdapterModelConfig, TransformersModelConfig, DeltaModelConfig, VLLMModelConfig, VLMTransformersModelConfig
+    ],
 ):
     if isinstance(config, AdapterModelConfig):
         model = AdapterModel(config=config)
@@ -197,6 +191,9 @@ def load_model_with_accelerate_or_default(
             raise ImportError(NO_VLLM_ERROR_MSG)
         model = VLLMModel(config=config)
         return model
+    elif isinstance(config, VLMTransformersModelConfig):
+        model = VLMTransformersModel(config=config)
+        return model
     else:
         model = TransformersModel(config=config)