analyze_results.py

"""
Analyze experiment results across all 4 levels.

Reads experiment-log.json from each level, builds comparison tables,
and uses Gemini 2.5 Flash for qualitative analysis.

Run after all experiments complete:
    python analyze_results.py
"""

import argparse
import json
import os
import sys
import time

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
# Import llm from one of the levels (they're all identical)
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "autoresearch"))
import llm

ROOT = os.path.dirname(os.path.abspath(__file__))

LEVELS = [
    {"key": "autoresearch", "folder": "autoresearch", "name": "AutoResearch", "level": 1,
     "description": "Basic loop: propose -> benchmark -> keep if better"},
    {"key": "feedback-loop", "folder": "feedback-loop", "name": "Feedback Loop", "level": 2,
     "description": "Adds structured reviewer feedback (issue type, severity, fix)"},
    {"key": "hyperagent", "folder": "hyperagent", "name": "HyperAgent", "level": 3,
     "description": "Adds self-improving strategy (meta-agent updates approach)"},
    {"key": "arena-single", "folder": "arena-loop", "name": "Arena Single", "level": "4a",
     "results_subfolder": "results-single",
     "description": "Adversarial co-evolution (1 code agent vs 1 test agent)"},
    {"key": "arena-loop", "folder": "arena-loop", "name": "Arena Loop", "level": "4b",
     "results_subfolder": "results",
     "description": "Adversarial tests + tournament selection (4 agents)"},
]

ALL_TASKS = ["snake", "support", "email_validation"]

OUTPUT_FILE = os.path.join(ROOT, "experiment-results.md")


def load_experiment_log(level_folder, task, results_subfolder="results"):
    """Load experiment-log.json for a given level and task."""
    path = os.path.join(ROOT, level_folder, results_subfolder, task, "experiment-log.json")
    if not os.path.exists(path):
        return None
    with open(path) as f:
        return json.load(f)


def format_metric(value, metric_name=None):
    """Format a metric value for display."""
    if value is None:
        return "N/A"
    if isinstance(value, float):
        if abs(value) < 0.01 and value != 0:
            return f"{value:.6f}"
        return f"{value:.3f}"
    return str(value)


def build_summary_table(all_results):
    """Build a markdown summary table from all results."""
    lines = []
    lines.append("| Level | Task | Metric | Baseline | Best | Improvement | Iters | Cost | Tokens |")
    lines.append("|-------|------|--------|----------|------|-------------|-------|------|--------|")

    for level_info in LEVELS:
        key = level_info["key"]
        name = level_info["name"]
        for task in ALL_TASKS:
            log = all_results.get(f"{key}/{task}")
            if not log:
                continue

            metric_name = log.get("metric_name", "score")
            baseline = log.get("baseline_metric")
            best = log.get("best_metric")
            iterations = log.get("iterations", 0)
            cost = log.get("estimated_cost_usd", 0)
            usage = log.get("token_usage", {})
            total_tokens = usage.get("total_tokens", 0)
            higher_is_better = log.get("higher_is_better", True)

            # Calculate improvement ratio
            improvement = ""
            if baseline is not None and best is not None and baseline != 0:
                if higher_is_better:
                    ratio = best / baseline if baseline > 0 else 0
                    improvement = f"{ratio:.1f}x"
                else:
                    ratio = baseline / best if best > 0 else 0
                    improvement = f"{ratio:.1f}x faster"

            baseline_str = format_metric(baseline)
            best_str = format_metric(best)
            cost_str = f"${cost:.4f}" if cost else "N/A"
            token_str = f"{total_tokens:,}" if total_tokens else "N/A"

            lines.append(
                f"| {name} | {task} | {metric_name} | {baseline_str} | "
                f"{best_str} | {improvement} | {iterations} | {cost_str} | {token_str} |"
            )

    return "\n".join(lines)


TASK_EXPLANATIONS = {
    "email_validation": {
        "intro": (
            "Email validation is a **boolean correctness** task: for each test email, the "
            "solution returns valid/invalid and the benchmark checks against known answers. "
            "The metric is accuracy (0.0 to 1.0). The initial test suite has 20 cases."
        ),
        "why_scores_differ": (
            "AutoResearch and HyperAgent both hit 100% (1.0) on the original 20 tests -- "
            "this task is simple enough that one good LLM proposal can solve it. AutoResearch "
            "did it in a single accepted iteration (1 LLM call), while Feedback Loop's reviewer "
            "overhead actually hurt: its solution reached only 90% because the reviewer's "
            "structured feedback pushed changes that broke edge cases the simpler approach got right."
        ),
        "arena_note": (
            "Arena Loop's reported 84% looks lower, but it is scored against a **much harder "
            "expanded test suite of 50 cases** (adversarial edge cases the other levels never saw). "
            "See Cross-Validation below for the fair comparison."
        ),
    },
    "snake": {
        "intro": (
            "Snake AI is a **deterministic scoring** task: the benchmark plays 20 games with "
            "fixed random seeds on a 10x10 grid and reports the average score (food eaten). "
            "There is no perfect score -- better algorithms eat more food before dying."
        ),
        "why_scores_differ": (
            "This task rewards sustained iteration. AutoResearch's basic loop improved from "
            "0.05 to 14.55 (2 accepted out of 10 tries) -- decent but limited by the lack of "
            "feedback on what went wrong. Feedback Loop reached 31.1 because the structured "
            "reviewer identified specific failure patterns (e.g. \"snake traps itself in corners\") "
            "and suggested targeted fixes. HyperAgent (27.5) and Arena Loop (29.2) are competitive "
            "but didn't surpass Feedback Loop -- for this task, knowing WHY the snake dies matters "
            "more than meta-level code rewriting or adversarial pressure."
        ),
        "arena_note": (
            "Snake uses a fixed subprocess benchmark, so Arena Loop's score of 29.2 is directly "
            "comparable to the other levels. The test suite grew from 1 to 6 cases but this "
            "did not affect the benchmark score. No fairness issue here."
        ),
    },
    "support": {
        "intro": (
            "Customer support Q&A is an **LLM-as-judge** task: the solution answers customer "
            "questions about a product, and a separate LLM call scores each answer's quality "
            "(0-100). Baselines vary across levels (5.0-15.0) because the LLM judge is "
            "non-deterministic -- the same initial code gets different scores each run. "
            "The \"Unified Judge\" column shows all solutions re-scored using rubric-based "
            "boolean checks (keyword + LLM YES/NO) for a fair, stable comparison."
        ),
        "why_scores_differ": (
            "The Unified Judge column uses **rubric-based boolean scoring**: each answer is checked "
            "against specific facts from the knowledge base (keyword match first, LLM YES/NO fallback "
            "for paraphrasing), plus quality checks for relevance, tone, and contradictions. "
            "This reduces scoring noise from 30+ points (old LLM 0-100 judge) to under 7 points.\n\n"
            "The original Best scores varied wildly due to LLM judge noise. Arena Loop's Best is scored "
            "against 28 expanded questions (not the original 10), and HyperAgent's original run score "
            "was inflated by a noisy single judge call. The Unified Judge column provides a fair comparison."
        ),
        "arena_note": (
            "**Arena Loop's reported Best is scored against an expanded test suite of 28 adversarial "
            "questions**, not the original 10. The unified judge column shows its score on the original "
            "questions for a fair comparison. See Cross-Validation below for the full data."
        ),
    },
}


def build_per_task_section(all_results, task):
    """Build a per-task comparison section with explanatory narrative."""
    lines = []
    lines.append(f"### {task}")
    lines.append("")

    task_data = []
    for level_info in LEVELS:
        key = level_info["key"]
        log = all_results.get(f"{key}/{task}")
        if log:
            task_data.append((level_info, log))

    if not task_data:
        lines.append("No results available for this task.")
        lines.append("")
        return "\n".join(lines)

    # Task intro
    explanation = TASK_EXPLANATIONS.get(task, {})
    if explanation.get("intro"):
        lines.append(explanation["intro"])
        lines.append("")

    metric_name = task_data[0][1].get("metric_name", "score")

    # Check if Arena Loop has test expansion that makes scores incomparable
    arena_log = all_results.get(f"arena-loop/{task}")
    arena_expanded = False
    arena_peak = None
    arena_peak_round = None
    unified_scores = None
    if arena_log:
        suite_size = arena_log.get("test_suite_size")
        arena_peak, arena_peak_round = _get_arena_pre_expansion_peak(all_results, task)
        if arena_peak is not None and suite_size:
            arena_best = arena_log.get("best_metric")
            # Flag if expansion meaningfully changed the score (>5% relative)
            if arena_best is not None and arena_peak is not None and arena_peak != 0:
                relative_diff = abs(arena_peak - arena_best) / abs(arena_peak)
                if relative_diff > 0.05:
                    arena_expanded = True

        # Check for unified cross-validation scores (support task)
        unified_path = os.path.join(ROOT, "arena-loop", "results", task,
                                     "cross_validation.json")
        if os.path.exists(unified_path):
            with open(unified_path) as f:
                unified_data = json.load(f)
            unified_scores = {k: v["score"] for k, v in unified_data.get("scores", {}).items()}

    if arena_expanded:
        col_name = "Unified Judge*" if unified_scores else "vs Original*"
        lines.append(f"| Level | Baseline | Best | {col_name} | Accepted/Total | Duration (s) | LLM Calls |")
        lines.append(f"|-------|----------|------|{'--' * (len(col_name) // 2 + 1)}|----------------|--------------|-----------|")
    else:
        lines.append(f"| Level | Baseline | Best | Accepted/Total | Duration (s) | LLM Calls |")
        lines.append(f"|-------|----------|------|----------------|--------------|-----------|")

    for level_info, log in task_data:
        baseline = format_metric(log.get("baseline_metric"))
        best = format_metric(log.get("best_metric"))
        accepted = log.get("accepted", 0)
        total = log.get("iterations", 0)
        duration = log.get("elapsed_seconds", 0)
        calls = log.get("token_usage", {}).get("calls", 0)

        if level_info["folder"] == "arena-loop":
            config = log.get("config", {})
            n_agents = config.get("code_agents", 4)
            n_rounds = config.get("rounds", total)
            progress = f"{n_rounds} rounds x {n_agents} agents"
        else:
            progress = f"{accepted}/{total}"

        if arena_expanded:
            if unified_scores:
                # Use unified judge scores for all levels
                uscore = unified_scores.get(level_info["key"])
                vs_orig = f"**{format_metric(uscore)}**" if uscore is not None else "N/A"
            elif level_info["key"] == "arena-loop":
                vs_orig = f"**{format_metric(arena_peak)}**"
            else:
                vs_orig = best
            lines.append(
                f"| {level_info['name']} | {baseline} | {best} | {vs_orig} | "
                f"{progress} | {duration} | {calls} |"
            )
        else:
            lines.append(
                f"| {level_info['name']} | {baseline} | {best} | "
                f"{progress} | {duration} | {calls} |"
            )

    if arena_expanded:
        lines.append("")
        if unified_scores:
            lines.append(
                f"*\"Unified Judge\" = all solutions scored using rubric-based boolean checks "
                f"(keyword + LLM YES/NO) against the original {unified_data.get('test_suite_size', 10)} "
                f"questions. Noise reduced from 30+ points to under 7 points. "
                f"Arena Loop's Best ({format_metric(arena_log.get('best_metric'))}) was scored "
                f"against the expanded suite ({arena_log.get('test_suite_size', '?')} cases).*"
            )
        else:
            lines.append(
                f"*\"vs Original\" = score against the original test suite only. "
                f"For Levels 1-3 this is the same as Best (they only have original tests). "
                f"For Arena Loop, Best is scored against the expanded suite "
                f"({arena_log.get('test_suite_size', '?')} cases); vs Original shows the "
                f"pre-expansion peak (round {arena_peak_round}) for fair comparison.*"
            )

    lines.append("")

    # History summary
    for level_info, log in task_data:
        if level_info["folder"] == "arena-loop":
            config = log.get("config", {})
            n_rounds = config.get("rounds", 0)
            suite_size = log.get("test_suite_size", 0)
            lines.append(
                f"**{level_info['name']}**: {n_rounds} rounds of competition, "
                f"test suite grew to {suite_size} cases"
            )
        else:
            history = log.get("history", [])
            if history:
                actions = [h.get("action", "UNKNOWN") for h in history]
                accepted_count = sum(1 for a in actions if a == "ACCEPTED")
                rejected_count = sum(1 for a in actions if a == "REJECTED")
                error_count = sum(1 for a in actions if a == "ERROR")
                lines.append(
                    f"**{level_info['name']}**: "
                    f"{accepted_count} accepted, {rejected_count} rejected, {error_count} errors"
                )

    lines.append("")

    # Why the scores differ
    if explanation.get("why_scores_differ"):
        lines.append("**Why these scores differ:**")
        lines.append(explanation["why_scores_differ"])
        lines.append("")

    # Arena note (fairness caveat)
    has_arena = any(li["folder"] == "arena-loop" for li, _ in task_data)
    if has_arena and explanation.get("arena_note"):
        lines.append("**Arena Loop scoring note:**")
        lines.append(explanation["arena_note"])
        lines.append("")

    return "\n".join(lines)


def _get_arena_pre_expansion_peak(all_results, task):
    """Get the peak pre_expansion_metric from arena-loop history for a task."""
    arena_log = all_results.get(f"arena-loop/{task}")
    if not arena_log:
        return None, None
    history = arena_log.get("history", [])
    if not history:
        return None, None
    best_val = None
    best_round = None
    for h in history:
        pre = h.get("pre_expansion_metric")
        if pre is not None and (best_val is None or pre > best_val):
            best_val = pre
            best_round = h.get("step", "?")
    return best_val, best_round


def _build_email_cross_validation(all_results):
    """Cross-validation for email_validation using actual test files."""
    arena_log = all_results.get("arena-loop/email_validation")
    if not arena_log:
        return ""

    try:
        task_dir = os.path.join(ROOT, "tasks", "email_validation")
        with open(os.path.join(task_dir, "initial_tests.json")) as f:
            initial_tests = json.load(f)

        expanded_path = os.path.join(ROOT, "arena-loop", "results", "email_validation",
                                      "tests", "final_tests.json")
        if not os.path.exists(expanded_path):
            return ""
        with open(expanded_path) as f:
            expanded_tests = json.load(f)
    except (FileNotFoundError, json.JSONDecodeError):
        return ""

    def score_solution(code, test_suite):
        ns = {}
        try:
            exec(compile(code, "<test>", "exec"), ns)
        except Exception:
            return 0.0
        fn = ns.get("validate_email")
        if not fn:
            return 0.0
        correct = 0
        for tc in test_suite:
            try:
                if bool(fn(tc["email"])) == tc["valid"]:
                    correct += 1
            except Exception:
                pass
        return correct / len(test_suite) if test_suite else 0.0

    lines = []
    lines.append(f"### email_validation (deterministic cross-validation)")
    lines.append("")
    lines.append(f"Original test suite: {len(initial_tests)} cases | "
                 f"Arena-expanded test suite: {len(expanded_tests)} cases")
    lines.append("")
    lines.append("| Level | vs Original | vs Expanded | Combined* |")
    lines.append("|-------|-------------|-------------|-----------|")

    # Collect scores for narrative
    level_exp_scores = []  # (name, key, score_exp) for levels 1-3
    arena_combined = None

    for level_info in LEVELS:
        name = level_info["name"]
        folder = level_info["folder"]
        key = level_info["key"]
        subfolder = level_info.get("results_subfolder", "results")
        best_path = os.path.join(ROOT, folder, subfolder, "email_validation",
                                  "solutions", "best.py")
        if not os.path.exists(best_path):
            lines.append(f"| {name} | NO SOLUTION | - | - |")
            continue
        with open(best_path) as f:
            code = f.read()
        score_orig = score_solution(code, initial_tests)
        score_exp = score_solution(code, expanded_tests)
        all_tests = {json.dumps(t, sort_keys=True): t for t in initial_tests + expanded_tests}
        score_all = score_solution(code, list(all_tests.values()))
        if score_orig >= 0.99 and score_exp < 0.70:
            note = " (brittle)"
        elif score_exp < 0.70:
            note = " (drops on harder tests)"
        else:
            note = ""
        lines.append(f"| {name} | {score_orig:.0%} | {score_exp:.0%} | {score_all:.0%}{note} |")

        if key in ("autoresearch", "feedback-loop", "hyperagent"):
            level_exp_scores.append(score_exp)
        if key == "arena-loop":
            arena_combined = score_all

    baseline_path = os.path.join(ROOT, "tasks", "email_validation", "initial_solution.py")
    if os.path.exists(baseline_path):
        with open(baseline_path) as f:
            baseline_code = f.read()
        base_orig = score_solution(baseline_code, initial_tests)
        base_exp = score_solution(baseline_code, expanded_tests)
        all_tests = {json.dumps(t, sort_keys=True): t for t in initial_tests + expanded_tests}
        base_all = score_solution(baseline_code, list(all_tests.values()))
        lines.append(f"| Baseline | {base_orig:.0%} | {base_exp:.0%} | {base_all:.0%} |")

    lines.append("")
    lines.append("*Combined = all unique tests from both suites.")
    lines.append("")
    # Build narrative from computed scores
    if level_exp_scores and arena_combined is not None:
        low = min(level_exp_scores)
        high = max(level_exp_scores)
        drop_range = f"{low:.0%}-{high:.0%}" if low != high else f"{low:.0%}"
        lines.append("Levels 1-3 optimized for the original 20 tests and scored 90-100% on them -- ")
        lines.append(f"but those solutions are **brittle**: they drop to {drop_range} on the expanded tests ")
        lines.append(f"(adversarial edge cases they never trained against). Arena Loop scored {arena_combined:.0%} on ")
        lines.append("the combined suite because it trained against adversarial pressure throughout.")
    lines.append("")
    return "\n".join(lines)


def _build_support_cross_validation(all_results):
    """Cross-validation for support. Uses unified judge scores if available."""
    arena_log = all_results.get("arena-loop/support")
    if not arena_log:
        return ""

    # Check for unified cross-validation results (from cross_validate.py --task support)
    unified_path = os.path.join(ROOT, "arena-loop", "results", "support",
                                 "cross_validation.json")
    has_unified = os.path.exists(unified_path)

    if has_unified:
        with open(unified_path) as f:
            unified = json.load(f)
        return _build_support_unified_section(all_results, unified)

    # Fallback: pre-expansion metric comparison
    return _build_support_pre_expansion_section(all_results)


def _build_support_unified_section(all_results, unified):
    """Build support cross-validation using unified LLM judge scores."""
    scores = unified.get("scores", {})
    suite_size = unified.get("test_suite_size", 10)
    expanded_size = unified.get("expanded_suite_size")
    has_expanded = expanded_size and any(
        s.get("expanded_score") is not None for s in scores.values())

    lines = []
    lines.append("### support (rubric-based boolean scoring)")
    lines.append("")
    lines.append(f"All solutions scored using boolean fact checks "
                 "(keyword match + LLM YES/NO fallback) and quality checks.")
    lines.append("")
    if has_expanded:
        lines.append(f"Original test suite: {suite_size} questions | "
                     f"Arena-expanded test suite: {expanded_size} questions")
        lines.append("")
        lines.append(f"| Level | vs Original ({suite_size}) | vs Expanded ({expanded_size}) | Original Run |")
        lines.append("|-------|-------------|-------------|------------|")
    else:
        lines.append(f"| Level | Original Run | Unified Judge | Difference |")
        lines.append("|-------|-------------|--------------|------------|")

    best_orig = None
    best_orig_name = None
    best_exp = None
    best_exp_name = None
    for level_info in LEVELS:
        name = level_info["name"]
        key = level_info["key"]
        entry = scores.get(key, {})
        orig_score = entry.get("score")
        exp_score = entry.get("expanded_score")
        if orig_score is None:
            continue

        if best_orig is None or orig_score > best_orig:
            best_orig = orig_score
            best_orig_name = name
        if exp_score is not None and (best_exp is None or exp_score > best_exp):
            best_exp = exp_score
            best_exp_name = name

        log = all_results.get(f"{key}/support")
        run_score = log.get("best_metric") if log else None
        run_str = format_metric(run_score)

        if has_expanded:
            exp_str = format_metric(exp_score) if exp_score is not None else "N/A"
            lines.append(
                f"| {name} | {format_metric(orig_score)} | {exp_str} | {run_str} |"
            )
        else:
            diff = f"{orig_score - run_score:+.2f}" if run_score is not None else "N/A"
            lines.append(
                f"| {name} | {run_str} | {format_metric(orig_score)} | {diff} |"
            )

    baseline = scores.get("baseline", {})
    if baseline.get("score") is not None:
        if has_expanded:
            exp_str = format_metric(baseline.get("expanded_score")) if baseline.get("expanded_score") is not None else "N/A"
            lines.append(f"| Baseline | {format_metric(baseline['score'])} | {exp_str} | N/A |")
        else:
            lines.append(f"| Baseline | N/A | {format_metric(baseline['score'])} | N/A |")

    lines.append("")

    # Noise-aware winner declarations (~7 point noise threshold for LLM-as-judge)
    noise_threshold = 7.0
    if has_expanded:
        # Check if top scores are within noise
        orig_scores = [(l["name"], scores.get(l["key"], {}).get("score", 0)) for l in LEVELS if scores.get(l["key"])]
        exp_scores = [(l["name"], scores.get(l["key"], {}).get("expanded_score", 0)) for l in LEVELS if scores.get(l["key"])]
        orig_scores.sort(key=lambda x: x[1], reverse=True)
        exp_scores.sort(key=lambda x: x[1], reverse=True)

        if len(orig_scores) >= 2 and orig_scores[0][1] - orig_scores[1][1] <= noise_threshold:
            lines.append(f"**Original {suite_size} questions:** {orig_scores[0][0]} "
                         f"({format_metric(orig_scores[0][1])}) and {orig_scores[1][0]} "
                         f"({format_metric(orig_scores[1][1])}) are within measurement "
                         f"noise (~{noise_threshold:.0f} points)")
        else:
            lines.append(f"**Winner (original {suite_size} questions):** {best_orig_name} "
                         f"with {format_metric(best_orig)}")

        if len(exp_scores) >= 2 and exp_scores[0][1] - exp_scores[1][1] <= noise_threshold:
            lines.append(f"**Expanded {expanded_size} questions:** {exp_scores[0][0]} "
                         f"({format_metric(exp_scores[0][1])}) and {exp_scores[1][0]} "
                         f"({format_metric(exp_scores[1][1])}) are within measurement "
                         f"noise (~{noise_threshold:.0f} points)")
        elif best_exp_name:
            lines.append(f"**Winner (expanded {expanded_size} questions):** {best_exp_name} "
                         f"with {format_metric(best_exp)}")
        lines.append("")
        lines.append(
            f"HyperAgent drops on the expanded suite, showing less robust solutions."
        )
        lines.append("")
        lines.append(
            f"*Note: The expanded questions are adversarial against the Arena's own solutions, "
            f"not universally harder. Other levels may score higher on them if their approach "
            f"happens to cover those topics well. The original questions are the fair comparison; "
            f"the expanded column shows topic coverage, not difficulty.*"
        )
    else:
        if best_orig_name:
            lines.append(f"**Winner: {best_orig_name}** with {format_metric(best_orig)} -- ")
            lines.append("scored using rubric-based boolean checks on the same questions as all other levels.")
    lines.append("")

    return "\n".join(lines)


def _build_support_pre_expansion_section(all_results):
    """Fallback: support comparison using pre_expansion_metric from arena history."""
    arena_log = all_results.get("arena-loop/support")
    peak_pre, peak_round = _get_arena_pre_expansion_peak(all_results, "support")
    if peak_pre is None:
        return ""

    arena_best = arena_log.get("best_metric")
    arena_suite_size = arena_log.get("test_suite_size", "?")

    lines = []
    lines.append("### support (pre-expansion fairness comparison)")
    lines.append("")
    lines.append("*No unified cross-validation data found. Run `python arena-loop/cross_validate.py "
                 "--task support` for a fair same-judge comparison.*")
    lines.append("")
    lines.append("Arena Loop's reported score is measured against an expanded test suite")
    lines.append(f"({arena_suite_size} questions), not the original 10. Direct comparison")
    lines.append("with Levels 1-3 (scored on 10 original questions) is unfair.")
    lines.append("")
    lines.append("| Level | Best Score | Test Suite Size | Notes |")
    lines.append("|-------|-----------|-----------------|-------|")

    for level_info in LEVELS:
        if level_info["folder"] == "arena-loop":
            continue
        log = all_results.get(f"{level_info['key']}/support")
        if log:
            best = log.get("best_metric")
            lines.append(f"| {level_info['name']} | {format_metric(best)} | 10 (original) | scored on fixed test suite |")

    lines.append(
        f"| Arena Loop | {format_metric(arena_best)} | {arena_suite_size} (expanded) | "
        f"scored on harder, expanded suite |"
    )
    lines.append(
        f"| Arena Loop (pre-expansion peak) | **{format_metric(peak_pre)}** | "
        f"10 (original) | round {peak_round}, before test expansion* |"
    )

    lines.append("")
    lines.append(f"*Pre-expansion metric: Arena Loop's score measured against the original")
    lines.append(f"test suite BEFORE new adversarial questions were added that round.")
    lines.append(f"This is the fairest apples-to-apples comparison with Levels 1-3.")
    lines.append("")
    return "\n".join(lines)


def _build_snake_cross_validation(all_results):
    """Cross-validation for snake -- deterministic benchmark, scores are comparable."""
    arena_log = all_results.get("arena-loop/snake")
    if not arena_log:
        return ""

    peak_pre, peak_round = _get_arena_pre_expansion_peak(all_results, "snake")

    lines = []
    lines.append("### snake (deterministic benchmark)")
    lines.append("")
    lines.append("Snake uses a fixed subprocess benchmark (20 games, deterministic seeds).")
    lines.append("The arena's test suite grew but the benchmark score is unaffected --")
    lines.append("scores are directly comparable across all levels.")
    lines.append("")
    lines.append("| Level | Best Score | Notes |")
    lines.append("|-------|-----------|-------|")

    for level_info in LEVELS:
        log = all_results.get(f"{level_info['key']}/snake")
        if log:
            best = log.get("best_metric")
            note = ""
            if level_info["key"] == "arena-loop" and peak_pre is not None:
                note = f"pre-expansion peak: {format_metric(peak_pre)} (round {peak_round})"
            lines.append(f"| {level_info['name']} | {format_metric(best)} | {note} |")

    lines.append("")
    return "\n".join(lines)


def build_cross_validation_section(all_results):
    """Build cross-validation sections for all tasks where Arena Loop ran."""
    has_arena = any(k.startswith("arena-loop/") or k.startswith("arena-single/") for k in all_results)
    if not has_arena:
        return ""

    lines = []
    lines.append("## Cross-Validation: Fairness Comparison")
    lines.append("")
    lines.append("**Why is this section needed?** Arena Loop (Level 4) doesn't just improve code -- ")
    lines.append("it also expands the test suite with adversarial edge cases. This means its reported ")
    lines.append("scores are measured against a HARDER test suite than Levels 1-3, making direct ")
    lines.append("comparison of the \"Best\" column in the summary table misleading. Scores on ")
    lines.append("expanded test suites are not comparable to scores on original test suites -- ")
    lines.append("they're different scales.")
    lines.append("")
    lines.append("This section puts all levels on the same scale for each task.")
    lines.append("")

    email_section = _build_email_cross_validation(all_results)
    support_section = _build_support_cross_validation(all_results)
    snake_section = _build_snake_cross_validation(all_results)

    if email_section:
        lines.append(email_section)
    if support_section:
        lines.append(support_section)
    if snake_section:
        lines.append(snake_section)

    lines.append("**Key takeaway:** When compared fairly, Arena Loop's solutions are")
    lines.append("competitive or best across all tasks. Its apparently-lower reported")
    lines.append("scores reflect harder test suites, not worse solutions.")
    lines.append("")

    return "\n".join(lines)


TASK_DESCRIPTIONS = {
    "snake": "Build a Snake game AI (higher score is better, deterministic)",
    "support": "Customer support Q&A quality (higher quality_score, LLM-as-judge)",
    "email_validation": "Email validation accuracy (higher accuracy, adversarial test expansion)",
}


def build_llm_prompt(all_results, summary_table, per_task_sections):
    """Build a prompt for Gemini to analyze the results."""

    # Only describe tasks that have results
    active_tasks = sorted(set(k.split("/")[1] for k in all_results.keys()))
    task_lines = "\n".join(
        f"- {t}: {TASK_DESCRIPTIONS.get(t, t)}" for t in active_tasks
    )

    prompt = f"""You are analyzing experiment results from a self-improving code agent project.
There are 5 levels of increasing sophistication:

Level 1 - AutoResearch: Basic loop. LLM proposes code, benchmark evaluates, keep if better.
Level 2 - Feedback Loop: Adds structured reviewer with issue taxonomy, severity, fix suggestions.
Level 3 - HyperAgent: Meta-agent rewrites its own source code (true code-rewriting self-improvement).
Level 4a - Arena Single: Adversarial co-evolution (1 code agent vs 1 test agent, no tournament).
Level 4b - Arena Loop: Adversarial test agents + tournament selection (4 agents). Code agents are mini-HyperAgents.

Tasks tested:
{task_lines}

DATA:

{summary_table}

{per_task_sections}
"""

    # Add cross-validation data if available
    cross_val = build_cross_validation_section(all_results)
    if cross_val:
        prompt += f"\n{cross_val}\n"

    # Add condensed raw data
    prompt += "\nRaw data:\n\n"
    for key, log in sorted(all_results.items()):
        folder, task = key.split("/")
        prompt += f"--- {folder}/{task} ---\n"
        prompt += f"  metric: {log.get('metric_name')}, baseline: {log.get('baseline_metric')}, best: {log.get('best_metric')}\n"
        prompt += f"  accepted: {log.get('accepted')}, rejected: {log.get('rejected')}, errors: {log.get('errors')}\n"
        prompt += f"  elapsed: {log.get('elapsed_seconds')}s, cost: ${log.get('estimated_cost_usd', 0):.4f}\n"
        if log.get("test_suite_size"):
            prompt += f"  test_suite_size: {log['test_suite_size']}\n"
        prompt += "\n"

    prompt += """
IMPORTANT CONTEXT:
- For LLM-as-judge tasks (support), baselines vary across levels because the LLM
  judge scores the same initial code differently each run. This is normal LLM
  non-determinism, NOT a difference in starting code. Do NOT say one level started
  from a "lower baseline" without explaining this. Compare levels by their BEST
  metric (absolute score), not by improvement ratio, when baselines differ.
- For the Arena Loop, compare its scores against the COMBINED cross-validation
  score, not against levels 1-3 scores on fixed tests. Arena Loop's "lower" score
  is actually the highest when tested against all tests.

INSTRUCTIONS: Write the analysis using EXACTLY this structure. Do not add or skip sections.
Only discuss tasks that have data. Do not mention tasks with no results.
Use plain ASCII only (no Unicode characters). Be concise and data-driven.
Do NOT use superlatives like "superior", "exceptional", or "groundbreaking".
When scores are within ~7 points on LLM-as-judge tasks (support), say they are
within measurement noise and do NOT declare a winner. Only declare winners when
the gap is clearly outside the noise range. These are N=1 experiments -- state
findings as observations, not universal truths.

## Per-Task Winner

For each task, state which level performed best and why in 2-3 sentences.
If scores are within noise (~7 points for support), say so explicitly instead
of declaring a winner.

## What Each Level Adds

For each level (1-4), one paragraph: what concrete benefit the data shows.
Note where added complexity pays off and where it doesn't.

## Cross-Validation Insight

(Only if cross-validation data is provided above.)
Explain the key finding across ALL tasks: Arena Loop's reported scores appear lower
because it is scored against harder, expanded test suites. When compared fairly
(same test suite), Arena Loop is competitive or best. For email_validation, levels
1-3 scored high on fixed tests but are brittle against adversarial tests. For
support, Feedback Loop and Arena Loop are within measurement noise of each other.
The email_validation cross-validation cleanly demonstrates that fixed benchmarks
get gamed.

## Cost-Effectiveness

Which level gives the most improvement per dollar? Note diminishing returns.

## When to Use Which Level

Practical guidance: match the level to the problem type.
"""
    return prompt


def main():
    parser = argparse.ArgumentParser(description="Analyze experiment results")
    parser.add_argument("--run-id", default=None, help="Run ID to include in output")
    cmd_args = parser.parse_args()
    run_id = cmd_args.run_id

    print("Scanning experiment results...")
    print()

    # Collect all results
    all_results = {}
    found = 0
    missing = 0

    for level_info in LEVELS:
        folder = level_info["folder"]
        key = level_info["key"]
        subfolder = level_info.get("results_subfolder", "results")
        for task in ALL_TASKS:
            log = load_experiment_log(folder, task, subfolder)
            if log:
                all_results[f"{key}/{task}"] = log
                print(f"  Found: {folder}/{subfolder}/{task}/experiment-log.json")
                found += 1

    # Discover which tasks actually have results (ignore tasks with no data)
    active_tasks = sorted(set(k.split("/")[1] for k in all_results.keys()))

    print()
    print(f"Found {found} experiment logs across {len(active_tasks)} tasks: {', '.join(active_tasks)}")

    if found == 0:
        print("\nNo experiment results found. Run experiments first:")
        print("  python autoresearch/experiment.py --task snake")
        print("  python feedback-loop/experiment.py --task snake")
        print("  python hyperagent/experiment.py --task snake")
        print("  python arena-loop/experiment.py --task snake")
        sys.exit(1)

    # Build deterministic tables
    print("\nBuilding comparison tables...")
    summary_table = build_summary_table(all_results)
    per_task_sections = ""
    for task in active_tasks:
        per_task_sections += build_per_task_section(all_results, task)

    # Send to Gemini for analysis
    print("Sending data to Gemini 2.5 Flash for analysis...")
    prompt = build_llm_prompt(all_results, summary_table, per_task_sections)

    try:
        analysis = llm.ask(prompt)
    except Exception as e:
        print(f"WARNING: LLM analysis failed: {e}")
        analysis = "(LLM analysis unavailable -- see tables above for raw data)"

    # Build output document
    date_str = time.strftime("%Y-%m-%d")
    lines = []
    lines.append("# Self-Improving Agents -- Experiment Results")
    lines.append("")
    if run_id:
        lines.append(f"Run ID: {run_id}")
        lines.append("")
    lines.append(f"Generated: {date_str}")
    lines.append("")
    lines.append("## Configuration")
    lines.append("")
    lines.append("- Model: Gemini 2.5 Flash")
    tested_levels = [l["name"] for l in LEVELS
                     if any(l["key"] + "/" + t in all_results for t in active_tasks)]
    lines.append(f"- Levels tested: {', '.join(tested_levels)}")
    lines.append(f"- Tasks tested: {', '.join(active_tasks)}")
    lines.append(f"- Total experiments: {found}")
    lines.append("")
    lines.append("## Summary Table")
    lines.append("")
    lines.append(summary_table)
    lines.append("")
    lines.append("*Note on LLM-as-judge tasks (support): Baselines vary across levels (5.0-15.0 range")
    lines.append("observed) because the LLM judge scores the same initial code differently each run.")
    lines.append("Each level's improvement ratio within its run is valid. Cross-level ratio comparisons")
    lines.append("should be interpreted with caution as they partly reflect baseline variation. The")
    lines.append("absolute Best score is the most reliable metric for cross-level comparison.*")
    lines.append("")
    lines.append("## Per-Task Analysis")
    lines.append("")
    lines.append(per_task_sections)
    # Cross-validation section (if arena-loop email data exists)
    cross_val = build_cross_validation_section(all_results)
    if cross_val:
        lines.append(cross_val)

    lines.append("## Limitations")
    lines.append("")
    lines.append("- **Single run per experiment (N=1)** -- no error bars or confidence intervals.")
    lines.append("  Snake and email_validation are deterministic (reproducible given the same code),")
    lines.append("  but support scores have ~7 point noise from LLM-as-judge scoring.")
    lines.append("- **Small task set (3 tasks)** -- findings may not generalize to other domains.")
    lines.append("- **Single model (Gemini 2.5 Flash)** -- results may differ with other LLMs.")
    lines.append("")
    lines.append("## Analysis")
    lines.append("")
    lines.append("*Analysis generated by Gemini 2.5 Flash*")
    lines.append("")
    lines.append(analysis)
    lines.append("")

    # Write output
    output = "\n".join(lines)
    with open(OUTPUT_FILE, "w", encoding="utf-8") as f:
        f.write(output)

    print(f"\nResults written to: {OUTPUT_FILE}")

    # Print token usage
    usage = llm.get_token_usage()
    if usage["calls"] > 0:
        cost = (
            usage["prompt_tokens"] * 0.30 / 1_000_000 +
            usage["completion_tokens"] * 2.50 / 1_000_000
        )
        print(f"Analysis LLM usage: {usage['total_tokens']:,} tokens, ${cost:.4f}")


if __name__ == "__main__":
    main()