Llm experiments

.vscode/dats.code-workspace

@@ -7,6 +7,10 @@
     {
       "name": "backend",
       "path": "../backend",
+    },
+        {
+      "name": "benchmarks",
+      "path": "../benchmarks",
     },
     {
       "name": "ray",

.vscode/settings.json

@@ -3,6 +3,7 @@
   "files.exclude": {
     "airflow": true,
     "backend": true,
+    "benchmarks": true,
     "ray": true,
     "frontend": true
   },

benchmarks/.gitignore

@@ -0,0 +1,14 @@
+.venv/
+__pycache__/
+*.pyc
+
+# Local env overrides
+docker/.env
+src/.env
+
+# Generated benchmark artifacts
+outputs/*.csv
+outputs/*.json
+outputs/*.png
+data/**/*.csv
+run_experiment.log

benchmarks/.vscode/launch.json

@@ -0,0 +1,95 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "fastapi",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${workspaceFolder}/src/main.py",
+      "console": "integratedTerminal",
+      "justMyCode": true,
+      "cwd": "${workspaceFolder}",
+      "envFile": "${workspaceFolder}/.env",
+      "env": {
+        "PYTHONPATH": "${workspaceFolder}/src"
+      }
+    },
+    {
+      "name": "rq",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${workspaceFolder}/src/worker.py",
+      "args": ["work", "dev"],
+      "console": "integratedTerminal",
+      "justMyCode": true,
+      "cwd": "${workspaceFolder}",
+      "envFile": "${workspaceFolder}/.env",
+      "env": {
+        "PYTHONPATH": "${workspaceFolder}/src",
+        "RQ_WORKERS_CPU": "1",
+        "RQ_WORKERS_API": "1",
+        "RQ_WORKERS_GPU": "1"
+      }
+    },
+    {
+      "name": "pytest",
+      "type": "debugpy",
+      "request": "launch",
+      "module": "pytest",
+      "console": "integratedTerminal",
+      "justMyCode": true,
+      "cwd": "${workspaceFolder}",
+      "envFile": "${workspaceFolder}/.env",
+      "env": {
+        "PYTHONPATH": "${workspaceFolder}/src",
+        "RESET_DATABASE_FOR_TESTING": "1"
+      }
+    },
+    {
+      "name": "pyright",
+      "type": "node-terminal",
+      "request": "launch",
+      "command": "uv run pyright",
+      "cwd": "${workspaceFolder}",
+      "envFile": "${workspaceFolder}/.env",
+      "env": {
+        "PYTHONPATH": "${workspaceFolder}/src"
+      }
+    },
+    {
+      "name": "Alembic: migrate",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${workspaceFolder}/src/migrations/run_migrations.py",
+      "console": "integratedTerminal",
+      "justMyCode": true,
+      "cwd": "${workspaceFolder}",
+      "envFile": "${workspaceFolder}/.env",
+      "env": {
+        "PYTHONPATH": "${workspaceFolder}/src"
+      }
+    },
+    {
+      "name": "Alembic: check",
+      "type": "node-terminal",
+      "request": "launch",
+      "command": "uv run alembic -c src/migrations/alembic.ini check",
+      "cwd": "${workspaceFolder}",
+      "envFile": "${workspaceFolder}/.env",
+      "env": {
+        "PYTHONPATH": "${workspaceFolder}/src"
+      }
+    },
+    {
+      "name": "Alembic: revision",
+      "type": "node-terminal",
+      "request": "launch",
+      "command": "uv run alembic -c src/migrations/alembic.ini revision --autogenerate -m \"vscode launcher\"",
+      "cwd": "${workspaceFolder}",
+      "envFile": "${workspaceFolder}/.env",
+      "env": {
+        "PYTHONPATH": "${workspaceFolder}/src"
+      }
+    }
+  ]
+}

benchmarks/.vscode/settings.json

@@ -0,0 +1,16 @@
+{
+  // python
+  "python.defaultInterpreterPath": "${workspaceFolder:benchmarks}/.venv/bin/python",
+  "python.envFile": "${workspaceFolder:benchmarks}/.env",
+  "python.autoComplete.extraPaths": ["${workspaceFolder:benchmarks}/src"],
+  "python.analysis.extraPaths": ["${workspaceFolder:benchmarks}/src"],
+  "python.analysis.pyrightVersion": "1.1.385", // this has to match pyproject.toml
+  "python.analysis.exclude": ["**/__pycache__", "**/.venv"],
+
+  // prettier
+  "prettier.prettierPath": "../frontend/node_modules/prettier",
+  "prettier.configPath": "../.prettierrc.yaml",
+
+  // ruff
+  "ruff.interpreter": ["${workspaceFolder:benchmarks}/.venv/bin/python"]
+}

benchmarks/README.md

@@ -0,0 +1,62 @@
+# LLM Benchmarking Framework
+
+This folder contains a modular and reproducible framework for benchmarking LLMs across NLP tasks.
+
+## First Working Experiment
+
+The first implemented end-to-end experiment is document classification on a sampled 20 Newsgroups split.
+
+Configuration rules:
+
+- Experiment and backend configs are composed via Hydra groups into typed `RunConfig` as `experiment` and `backend`.
+- Model config is nested in each experiment via `defaults` (`/model: ...`) as `experiment.model`.
+- Dataset config is nested in each experiment via `defaults` (`/dataset: ...`) as `experiment.dataset`.
+- Dataset configs define `name`, `path`, `text_column`, and `label_column`.
+- `run_name` in experiment configs is optional. If omitted, MLflow auto-generates it.
+- Prompt templates are always loaded from `src/prompts/templates` (not configurable).
+- Schema is configured as a single dotted path (for example `newsgroups20_schema.NewsgroupClassificationSchemaV1`).
+
+### 1. Install dependencies
+
+```bash
+cd benchmarks
+uv sync
+```
+
+### 2. Prepare data
+
+```bash
+uv run python data/20newsgroups/preprocess.py
+```
+
+### 3. Start MLflow service
+
+```bash
+cd docker
+cp .env.example .env
+docker compose up -d
+cd ..
+```
+
+### 4. Run the 20 Newsgroups experiment
+
+```bash
+uv run python src/run_experiment.py
+```
+
+### 5. Override config groups (example)
+
+```bash
+uv run python src/run_experiment.py \
+	experiment=20newsgroups_v1_zeroshot \
+	backend=vllm \
+	backend.gpu_id=1
+```
+
+## Layout
+
+- `configs/`: Runtime config and Hydra groups (`experiment/`, `model/`, `dataset/`, `backend/`)
+- `data/`: Datasets and preprocessing scripts
+- `docker/`: MLflow compose files and environment templates
+- `outputs/`: Local output artifacts (CSV/JSON)
+- `src/`: Core runner, LLM clients, schemas, prompts, evaluation, tracking

benchmarks/configs/backend/vllm.yaml

@@ -0,0 +1,8 @@
+image: vllm/vllm-openai:latest
+host_port: 19275
+startup_timeout_seconds: 600
+gpu_id: 1
+hf_token_env_var: HF_TOKEN
+hf_cache_dir: ~/.cache/huggingface
+concurrency: 24
+api_key: EMPTY

benchmarks/configs/config.yaml

@@ -0,0 +1,15 @@
+defaults:
+  - experiment: 20newsgroups_v1_zeroshot
+  - backend: vllm
+  - _self_
+
+output_dir: outputs
+mlflow_uri: http://localhost:19274
+fail_on_parse_error: false
+
+hydra:
+  job:
+    chdir: false
+  run:
+    dir: .
+  output_subdir: null

benchmarks/configs/dataset/20newsgroups.yaml

@@ -0,0 +1,5 @@
+name: 20newsgroups
+dataset_type: document_classification_single_label
+path: 20newsgroups/test_full_raw.parquet
+text_column: document_text
+label_column: label

benchmarks/configs/dataset/bbc-coarse.yaml

@@ -0,0 +1,5 @@
+name: bbc-coarse
+dataset_type: document_classification_single_label
+path: bbc/bbc_cleaned.parquet
+text_column: content
+label_column: main_tag

benchmarks/configs/dataset/bbc-fine.yaml

@@ -0,0 +1,5 @@
+name: bbc-fine
+dataset_type: document_classification_single_label
+path: bbc/bbc_cleaned.parquet
+text_column: content
+label_column: tag

benchmarks/configs/dataset/coarsediscourse.yaml

@@ -0,0 +1,5 @@
+name: coarsediscourse
+dataset_type: sequential_sentence_classification
+path: coarsediscourse/coursediscourse_test.parquet
+sentences_column: sentences
+labels_column: labels

benchmarks/configs/dataset/csabstruct.yaml

@@ -0,0 +1,5 @@
+name: csabstruct
+dataset_type: sequential_sentence_classification
+path: csabstruct/test.parquet
+sentences_column: sentences
+labels_column: labels

benchmarks/configs/dataset/daily-dialog.yaml

@@ -0,0 +1,5 @@
+name: daily-dialog
+dataset_type: sequential_sentence_classification
+path: daily_dialog/dailydialog_test.parquet
+sentences_column: sentences
+labels_column: labels

benchmarks/configs/dataset/emotion-lines.yaml

@@ -0,0 +1,7 @@
+name: emotion-lines
+dataset_type: sequential_sentence_classification
+path: emotion_lines/friends_test.parquet
+sentences_column: sentences
+labels_column: labels
+unwanted_labels:
+  - non-neutral

benchmarks/configs/dataset/fewnerd-coarse.yaml

@@ -0,0 +1,15 @@
+name: fewnerd-coarse
+dataset_type: span_classification
+path: fewnerd/fewnerd_test.parquet
+tokens_column: tokens
+tags_column: ner_tags
+id2label:
+  0: O
+  1: art
+  2: building
+  3: event
+  4: location
+  5: organization
+  6: other
+  7: person
+  8: product

benchmarks/configs/dataset/fewnerd-fine.yaml

@@ -0,0 +1,73 @@
+name: fewnerd-fine
+dataset_type: span_classification
+path: fewnerd/fewnerd_test.parquet
+tokens_column: tokens
+tags_column: fine_ner_tags
+id2label:
+  0: O
+  1: art - broadcastprogram
+  2: art - film
+  3: art - music
+  4: art - other
+  5: art - painting
+  6: art - writtenart
+  7: building - airport
+  8: building - hospital
+  9: building - hotel
+  10: building - library
+  11: building - other
+  12: building - restaurant
+  13: building - sportsfacility
+  14: building - theater
+  15: event - attack/battle/war/militaryconflict
+  16: event - disaster
+  17: event - election
+  18: event - other
+  19: event - protest
+  20: event - sportsevent
+  21: location - GPE
+  22: location - bodiesofwater
+  23: location - island
+  24: location - mountain
+  25: location - other
+  26: location - park
+  27: location - road/railway/highway/transit
+  28: organization - company
+  29: organization - education
+  30: organization - government/governmentagency
+  31: organization - media/newspaper
+  32: organization - other
+  33: organization - politicalparty
+  34: organization - religion
+  35: organization - showorganization
+  36: organization - sportsleague
+  37: organization - sportsteam
+  38: other - astronomything
+  39: other - award
+  40: other - biologything
+  41: other - chemicalthing
+  42: other - currency
+  43: other - disease
+  44: other - educationaldegree
+  45: other - god
+  46: other - language
+  47: other - law
+  48: other - livingthing
+  49: other - medical
+  50: person - actor
+  51: person - artist/author
+  52: person - athlete
+  53: person - director
+  54: person - other
+  55: person - politician
+  56: person - scholar
+  57: person - soldier
+  58: product - airplane
+  59: product - car
+  60: product - food
+  61: product - game
+  62: product - other
+  63: product - ship
+  64: product - software
+  65: product - train
+  66: product - weapon

benchmarks/configs/dataset/german-ler-coarse.yaml

@@ -0,0 +1,14 @@
+name: german-ler-coarse
+dataset_type: span_classification
+path: german-ler/german_ler_test.parquet
+tokens_column: tokens
+tags_column: ner_tags
+id2label:
+  0: O
+  1: person
+  2: ort
+  3: organisation
+  4: norm
+  5: gesetz
+  6: rechtsprechung
+  7: literatur