Phase 4: eval ablation runner — 3-config comparison

Adds the 3-config ablation eval that measures benefit of each layer:
- Config 1: Single Groq call — baseline (no multi-agent, no MCP)
- Config 2: Multi-agent without MCP (routing + parallel annotation)
- Config 3: Full pipeline — multi-agent + Redis examples + guidelines

eval/evaluators.py  — EvalResult/EvalSummary dataclasses, compute_summary,
                      print_comparison with MAFA/ML reference baselines
eval/run_eval.py    — CLI runner (--smoke --n --configs --output)
tests/test_eval.py  — 10 unit tests for metric functions (no API calls)

Also:
- Add use_mcp=True param to build_annotation_context and annotate/validate
  so Config 2 can run agents without Redis/guidelines lookup
- Fix Makefile eval-smoke to use python -m eval.run_eval

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: saviolobo

Makefile

@@ -15,4 +15,4 @@ format:
 	uv run ruff check --fix .
 
 eval-smoke:
-	uv run python eval/run_eval.py --smoke --n 10
+	uv run python -m eval.run_eval --smoke --n 10

agents/primary_annotator.py

@@ -59,11 +59,16 @@ def label_must_be_valid(cls, v: str) -> str:
         return v
 
 
-def build_annotation_context(query: str) -> str:
+def build_annotation_context(query: str, use_mcp: bool = True) -> str:
     """Build the shared user prompt used by Primary and Validator agents."""
     valid = list_valid_intents()
     intent_list = "\n".join(f"- {e['intent_name']}" for e in valid)
 
+    if not use_mcp:
+        return (
+            f"Customer query: {query}\n\nValid intent labels (choose exactly one):\n{intent_list}"
+        )
+
     similar = search_similar_examples(query, k=5)
     examples_block = "\n".join(f'  "{ex["text"]}" → {ex["intent_name"]}' for ex in similar)
 
@@ -109,9 +114,9 @@ def _get_client() -> Cerebras:
     wait=wait_exponential(multiplier=1, min=2, max=30),
     stop=stop_after_attempt(3),
 )
-def annotate(query: str) -> AnnotatorOutput:
+def annotate(query: str, use_mcp: bool = True) -> AnnotatorOutput:
     """Assign an intent label to a banking customer query."""
-    user_prompt = build_annotation_context(query)
+    user_prompt = build_annotation_context(query, use_mcp=use_mcp)
     response = _get_client().chat.completions.create(
         model="qwen-3-235b-a22b-instruct-2507",
         messages=[

agents/validator.py

@@ -54,13 +54,13 @@ def _get_client() -> Cerebras:
     wait=wait_exponential(multiplier=1, min=2, max=30),
     stop=stop_after_attempt(3),
 )
-def validate(query: str) -> AnnotatorOutput:
+def validate(query: str, use_mcp: bool = True) -> AnnotatorOutput:
     """Independently assign an intent label — blind to any prior annotation."""
     response = _get_client().chat.completions.create(
         model="qwen-3-235b-a22b-instruct-2507",
         messages=[
             {"role": "system", "content": _SYSTEM},
-            {"role": "user", "content": build_annotation_context(query)},
+            {"role": "user", "content": build_annotation_context(query, use_mcp=use_mcp)},
         ],
         response_format={"type": "json_object"},
         temperature=0,

eval/evaluators.py

@@ -0,0 +1,79 @@
+"""Evaluation metrics for the annotation pipeline."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from sklearn.metrics import f1_score
+
+
+@dataclass
+class EvalResult:
+    query_id: str
+    predicted_label: str
+    true_label: str
+    route_to_human: bool = False
+    elapsed_seconds: float = 0.0
+
+
+@dataclass
+class EvalSummary:
+    config_name: str
+    n_items: int
+    agreement_rate: float
+    macro_f1: float
+    human_review_rate: float
+    items_per_hour: float
+    total_elapsed_seconds: float
+
+
+def compute_summary(
+    config_name: str,
+    results: list[EvalResult],
+    total_elapsed: float,
+) -> EvalSummary:
+    n = len(results)
+    if n == 0:
+        raise ValueError("No results to summarize")
+
+    human_count = sum(1 for r in results if r.route_to_human)
+    correct = sum(1 for r in results if not r.route_to_human and r.predicted_label == r.true_label)
+
+    # Human-routed items use a sentinel that won't match any true label
+    preds = ["__human__" if r.route_to_human else r.predicted_label for r in results]
+    trues = [r.true_label for r in results]
+    mf1 = float(f1_score(trues, preds, average="macro", zero_division=0))
+
+    return EvalSummary(
+        config_name=config_name,
+        n_items=n,
+        agreement_rate=correct / n,
+        macro_f1=mf1,
+        human_review_rate=human_count / n,
+        items_per_hour=n / total_elapsed * 3600 if total_elapsed > 0 else 0.0,
+        total_elapsed_seconds=total_elapsed,
+    )
+
+
+def print_comparison(summaries: list[EvalSummary]) -> None:
+    w = 35
+    header = (
+        f"{'Config':<{w}} {'N':>6} {'Agree%':>8} {'Macro-F1':>10} {'Human%':>8} {'items/hr':>10}"
+    )
+    print()
+    print(header)
+    print("-" * len(header))
+    for s in summaries:
+        print(
+            f"{s.config_name:<{w}} {s.n_items:>6} "
+            f"{s.agreement_rate * 100:>7.1f}% "
+            f"{s.macro_f1:>10.4f} "
+            f"{s.human_review_rate * 100:>7.1f}% "
+            f"{s.items_per_hour:>10.1f}"
+        )
+    print()
+    print("Reference baselines:")
+    print("  JP Morgan MAFA (AAAI 2026)   86.0%  agreement")
+    print("  Best supervised ML           87.35% Macro-F1")
+    print("  Manual annotation cost       ~$0.15/item")
+    print()

eval/run_eval.py

@@ -0,0 +1,266 @@
+"""Ablation runner — 3-config evaluation of the annotation pipeline.
+
+Config 1: Single LLM call, no multi-agent, no MCP (baseline)
+Config 2: Multi-agent without MCP (structural benefit only)
+Config 3: Full pipeline — multi-agent + MCP guidelines + Redis examples
+
+Usage:
+  uv run python eval/run_eval.py --smoke --n 10
+  uv run python eval/run_eval.py --configs 3 --n 50
+  uv run python eval/run_eval.py --output results.json
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import time
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+FIXTURES_DIR = Path(__file__).parent.parent / "tests" / "fixtures" / "items"
+HOLDOUT_PATH = Path(__file__).parent.parent / "data" / "splits" / "holdout.parquet"
+
+
+# ---------- item loading ----------
+
+
+def _load_smoke_items(n: int) -> list[dict]:
+    items = [json.loads(f.read_text()) for f in sorted(FIXTURES_DIR.glob("item_*.json"))]
+    return items[:n]
+
+
+def _load_holdout_items(n: int | None) -> list[dict]:
+    if not HOLDOUT_PATH.exists():
+        print(
+            f"ERROR: {HOLDOUT_PATH} not found. Run eval/datasets.py first.",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+    import pandas as pd
+
+    df = pd.read_parquet(HOLDOUT_PATH)
+    if n:
+        df = df.head(n)
+    return [
+        {"query_id": row.query_id, "text": row.text, "true_intent": row.category}
+        for row in df.itertuples()
+    ]
+
+
+# ---------- Config 1: single LLM ----------
+
+_SINGLE_LLM_SYSTEM = """You are a banking customer service intent classifier.
+Assign exactly one intent label from the provided list.
+Respond with ONLY valid JSON: {"label": "<intent_name>"}"""
+
+
+def _run_config1(items: list[dict]) -> tuple[list, float]:
+    from groq import Groq
+
+    from eval.evaluators import EvalResult
+    from mcp_servers.label_schema_mcp.server import list_valid_intents
+
+    client = Groq()
+    valid = {e["intent_name"] for e in list_valid_intents()}
+    intent_list = "\n".join(f"- {n}" for n in sorted(valid))
+
+    results: list[EvalResult] = []
+    t_start = time.perf_counter()
+
+    for i, item in enumerate(items, 1):
+        print(f"  [1/{len(items)}→{i}] {item['query_id']}", end=" ", flush=True)
+        t0 = time.perf_counter()
+        try:
+            resp = client.chat.completions.create(
+                model="llama-3.1-8b-instant",
+                messages=[
+                    {"role": "system", "content": _SINGLE_LLM_SYSTEM},
+                    {
+                        "role": "user",
+                        "content": (
+                            f"Customer query: {item['text']}\n\nValid intents:\n{intent_list}"
+                        ),
+                    },
+                ],
+                response_format={"type": "json_object"},
+                temperature=0,
+            )
+            raw = json.loads(resp.choices[0].message.content)
+            label = raw.get("label", "").strip()
+            if label not in valid:
+                label = "__invalid__"
+        except Exception as exc:
+            print(f"[ERROR: {exc}]", end=" ")
+            label = "__error__"
+
+        elapsed = time.perf_counter() - t0
+        print(f"→ {label} ({elapsed:.1f}s)")
+        results.append(
+            EvalResult(
+                query_id=item["query_id"],
+                predicted_label=label,
+                true_label=item["true_intent"],
+                elapsed_seconds=elapsed,
+            )
+        )
+
+    return results, time.perf_counter() - t_start
+
+
+# ---------- Config 2: multi-agent, no MCP ----------
+
+
+def _run_config2(items: list[dict]) -> tuple[list, float]:
+    from agents.arbitrator import CONFIDENCE_THRESHOLD, arbitrate
+    from agents.primary_annotator import annotate
+    from agents.router import route as router_route
+    from agents.validator import agrees_with, validate
+    from eval.evaluators import EvalResult
+
+    results: list[EvalResult] = []
+    t_start = time.perf_counter()
+
+    for i, item in enumerate(items, 1):
+        print(f"  [2/{len(items)}→{i}] {item['query_id']}", end=" ", flush=True)
+        t0 = time.perf_counter()
+        query = item["text"]
+
+        route_decision = router_route(query)
+        primary = annotate(query, use_mcp=False)
+        final_label = primary.label
+        route_to_human = False
+
+        if route_decision.route == "COMPLEX":
+            validator = validate(query, use_mcp=False)
+            if not agrees_with(validator, primary.label):
+                arb = arbitrate(query, primary, validator)
+                final_label = arb.final_label
+                route_to_human = arb.confidence < CONFIDENCE_THRESHOLD
+
+        elapsed = time.perf_counter() - t0
+        status = "→human" if route_to_human else f"→ {final_label}"
+        print(f"{route_decision.route} {status} ({elapsed:.1f}s)")
+        results.append(
+            EvalResult(
+                query_id=item["query_id"],
+                predicted_label=final_label,
+                true_label=item["true_intent"],
+                route_to_human=route_to_human,
+                elapsed_seconds=elapsed,
+            )
+        )
+
+    return results, time.perf_counter() - t_start
+
+
+# ---------- Config 3: full pipeline ----------
+
+
+def _run_config3(items: list[dict]) -> tuple[list, float]:
+    from eval.evaluators import EvalResult
+    from graph.pipeline import build_pipeline
+
+    pipeline = build_pipeline()
+    results: list[EvalResult] = []
+    t_start = time.perf_counter()
+
+    for i, item in enumerate(items, 1):
+        print(f"  [3/{len(items)}→{i}] {item['query_id']}", end=" ", flush=True)
+        t0 = time.perf_counter()
+        state = pipeline.invoke(
+            {
+                "query_id": item["query_id"],
+                "query": item["text"],
+                "batch_stats": {},
+            }
+        )
+        elapsed = time.perf_counter() - t0
+        final_label = state.get("final_label", "__none__")
+        route_to_human = bool(state.get("route_to_human"))
+        status = "→human" if route_to_human else f"→ {final_label}"
+        print(f"{state.get('route', '?')} {status} ({elapsed:.1f}s)")
+        results.append(
+            EvalResult(
+                query_id=item["query_id"],
+                predicted_label=final_label,
+                true_label=item["true_intent"],
+                route_to_human=route_to_human,
+                elapsed_seconds=elapsed,
+            )
+        )
+
+    return results, time.perf_counter() - t_start
+
+
+# ---------- main ----------
+
+_CONFIG_NAMES = {
+    "1": "Config 1: Single LLM (baseline)",
+    "2": "Config 2: Multi-agent, no MCP",
+    "3": "Config 3: Full pipeline (MCP)",
+}
+
+_CONFIG_FNS = {
+    "1": _run_config1,
+    "2": _run_config2,
+    "3": _run_config3,
+}
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Run ablation eval on the annotation pipeline")
+    parser.add_argument("--smoke", action="store_true", help="Use fixture items instead of holdout")
+    parser.add_argument("--n", type=int, default=10, help="Number of items to evaluate")
+    parser.add_argument(
+        "--configs",
+        nargs="+",
+        choices=["1", "2", "3"],
+        default=["1", "2", "3"],
+        metavar="N",
+        help="Which configs to run (default: 1 2 3)",
+    )
+    parser.add_argument("--output", type=str, default=None, help="Save results to JSON file")
+    args = parser.parse_args()
+
+    if args.smoke:
+        items = _load_smoke_items(args.n)
+        print(f"Smoke mode: {len(items)} fixture items")
+    else:
+        items = _load_holdout_items(args.n)
+        print(f"Holdout mode: {len(items)} items")
+
+    if not items:
+        print("No items to evaluate.", file=sys.stderr)
+        sys.exit(1)
+
+    from eval.evaluators import compute_summary, print_comparison
+
+    all_results: dict[str, list] = {}
+    all_summaries = []
+
+    for cfg in args.configs:
+        print(f"\nRunning {_CONFIG_NAMES[cfg]} ...")
+        results, elapsed = _CONFIG_FNS[cfg](items)
+        summary = compute_summary(_CONFIG_NAMES[cfg], results, elapsed)
+        all_results[cfg] = [vars(r) for r in results]
+        all_summaries.append(summary)
+
+    print_comparison(all_summaries)
+
+    if args.output:
+        out_path = Path(args.output)
+        payload = {
+            "summaries": [vars(s) for s in all_summaries],
+            "per_config_results": all_results,
+        }
+        out_path.write_text(json.dumps(payload, indent=2))
+        print(f"Results saved to {out_path}")
+
+
+if __name__ == "__main__":
+    main()