feat: build Validator Agent (blind second-pass) and 12 fixture tests

- validator.py: independent annotation using same Cerebras model + MCP context
  but strictly blind to Primary output; agrees_with() helper for comparison
- primary_annotator.py: rename _build_user_prompt → build_annotation_context (public)
  so both agents share identical context-building without duplication
- 12/12 tests green including cross-agent agreement test on unambiguous query

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: saviolobo

agents/primary_annotator.py

@@ -59,7 +59,8 @@ def label_must_be_valid(cls, v: str) -> str:
         return v
 
 
-def _build_user_prompt(query: str) -> str:
+def build_annotation_context(query: str) -> str:
+    """Build the shared user prompt used by Primary and Validator agents."""
     valid = list_valid_intents()
     intent_list = "\n".join(f"- {e['intent_name']}" for e in valid)
 
@@ -112,7 +113,7 @@ def _get_client() -> Cerebras:
 )
 def annotate(query: str) -> AnnotatorOutput:
     """Assign an intent label to a banking customer query."""
-    user_prompt = _build_user_prompt(query)
+    user_prompt = build_annotation_context(query)
     response = _get_client().chat.completions.create(
         model="qwen-3-235b-a22b-instruct-2507",
         messages=[

agents/validator.py

@@ -0,0 +1,74 @@
+"""Validator Agent — independent second-pass annotation (blind to Primary).
+
+Model: qwen-3-235b-a22b-instruct-2507 via Cerebras
+Key constraint: receives the same query + guidelines as Primary, but NEVER sees
+Primary's label or reasoning. Agreement → high-confidence annotation.
+Disagreement → routes to Arbitrator.
+Output: {label, confidence, reasoning}
+"""
+
+import json
+
+from cerebras.cloud.sdk import Cerebras
+from cerebras.cloud.sdk._exceptions import RateLimitError
+from dotenv import load_dotenv
+from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential
+
+from agents.primary_annotator import AnnotatorOutput, build_annotation_context
+
+load_dotenv()
+
+_SYSTEM = """You are an independent banking customer service intent classifier.
+
+IMPORTANT: You are performing a blind second annotation. You have not seen any prior
+classification of this query and must form your own independent judgment.
+
+You will be given:
+1. A customer query
+2. The list of all 77 valid Banking77 intent labels
+3. Similar labeled examples retrieved from the training set
+4. Detailed guidelines for the most likely candidate intents
+
+Your job: assign EXACTLY ONE label from the 77 valid intents independently.
+
+Rules:
+- The label must be copied exactly from the valid intents list (including casing and punctuation)
+- Confidence 0.9+ means you are certain; below 0.6 means the case is genuinely ambiguous
+- Reasoning: 2-3 sentences explaining your choice and why you rejected close alternatives
+
+Respond with ONLY this JSON (no markdown):
+{"label": "...", "confidence": 0.85, "reasoning": "...", "evidence": "..."}"""
+
+_client: Cerebras | None = None
+
+
+def _get_client() -> Cerebras:
+    global _client
+    if _client is None:
+        _client = Cerebras()
+    return _client
+
+
+@retry(
+    retry=retry_if_exception_type(RateLimitError),
+    wait=wait_exponential(multiplier=1, min=2, max=30),
+    stop=stop_after_attempt(3),
+)
+def validate(query: str) -> AnnotatorOutput:
+    """Independently assign an intent label — blind to any prior annotation."""
+    response = _get_client().chat.completions.create(
+        model="qwen-3-235b-a22b-instruct-2507",
+        messages=[
+            {"role": "system", "content": _SYSTEM},
+            {"role": "user", "content": build_annotation_context(query)},
+        ],
+        response_format={"type": "json_object"},
+        temperature=0,
+    )
+    raw = json.loads(response.choices[0].message.content)
+    return AnnotatorOutput(**raw)
+
+
+def agrees_with(validator_output: AnnotatorOutput, primary_label: str) -> bool:
+    """Return True if Validator's label matches Primary's label."""
+    return validator_output.label == primary_label

tests/test_validator.py

@@ -0,0 +1,80 @@
+"""Validator Agent tests — runs against live Cerebras + Redis APIs.
+
+Skipped automatically if CEREBRAS_API_KEY is not set.
+"""
+
+import json
+import os
+from pathlib import Path
+
+import pytest
+
+requires_cerebras = pytest.mark.skipif(
+    not os.getenv("CEREBRAS_API_KEY"), reason="CEREBRAS_API_KEY not set"
+)
+
+FIXTURES_DIR = Path(__file__).parent / "fixtures" / "items"
+
+
+def _load_fixtures(difficulty: str) -> list[dict]:
+    return [
+        json.loads(f.read_text())
+        for f in sorted(FIXTURES_DIR.glob("item_*.json"))
+        if json.loads(f.read_text())["difficulty"] == difficulty
+    ]
+
+
+SIMPLE_ITEMS = _load_fixtures("simple")
+AMBIGUOUS_ITEMS = _load_fixtures("ambiguous")
+
+
+@requires_cerebras
+@pytest.mark.parametrize("item", SIMPLE_ITEMS, ids=[i["true_intent"] for i in SIMPLE_ITEMS])
+def test_simple_items_get_correct_label(item: dict) -> None:
+    from agents.validator import validate
+
+    result = validate(item["text"])
+    assert result.label == item["true_intent"], (
+        f"text: {item['text']!r}\n"
+        f"expected: {item['true_intent']}, got: {result.label}\n"
+        f"reasoning: {result.reasoning}"
+    )
+    assert result.confidence >= 0.7
+
+
+@requires_cerebras
+@pytest.mark.parametrize("item", AMBIGUOUS_ITEMS, ids=[i["true_intent"] for i in AMBIGUOUS_ITEMS])
+def test_ambiguous_items_return_valid_label(item: dict) -> None:
+    from agents.validator import validate
+    from mcp_servers.label_schema_mcp.server import list_valid_intents
+
+    valid = {e["intent_name"] for e in list_valid_intents()}
+    result = validate(item["text"])
+    assert result.label in valid
+    assert 0.0 <= result.confidence <= 1.0
+    assert len(result.reasoning) > 10
+
+
+@requires_cerebras
+def test_agrees_with_helper() -> None:
+    from agents.validator import agrees_with, validate
+
+    result = validate("My card got lost.")
+    assert result.label == "lost_or_stolen_card"
+    assert agrees_with(result, "lost_or_stolen_card") is True
+    assert agrees_with(result, "compromised_card") is False
+
+
+@requires_cerebras
+def test_validator_agrees_with_primary_on_simple_query() -> None:
+    """Both agents should independently reach the same label on a clear query."""
+    from agents.primary_annotator import annotate
+    from agents.validator import agrees_with, validate
+
+    query = "I need to change my PIN."
+    primary = annotate(query)
+    validator = validate(query)
+    assert agrees_with(validator, primary.label), (
+        f"Primary: {primary.label}, Validator: {validator.label} — "
+        "expected agreement on an unambiguous query"
+    )