-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathclimate_evidence_report.html
More file actions
5 lines (5 loc) · 8.46 KB
/
Copy pathclimate_evidence_report.html
File metadata and controls
5 lines (5 loc) · 8.46 KB
1
2
3
4
5
<!doctype html><html lang="en"><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>Climate Evidence Bench</title>
<style>body{font:16px system-ui;max-width:1050px;margin:2rem auto;padding:0 1rem;color:#17231f}.notice{background:#fff3cd;border-left:4px solid #d49a00;padding:1rem}.cards{display:grid;grid-template-columns:repeat(auto-fit,minmax(170px,1fr));gap:1rem;margin:1.5rem 0}.card{border:1px solid #d3ddd8;border-radius:10px;padding:1rem}.value{font-size:1.5rem;font-weight:700}.tables{display:grid;grid-template-columns:1fr 2fr;gap:2rem}table{border-collapse:collapse;width:100%}th,td{padding:.5rem;border-bottom:1px solid #dce4e0;text-align:left}code{word-break:break-all}@media(max-width:750px){.tables{grid-template-columns:1fr}}</style>
<body><h1>Climate Evidence Bench</h1><p class="notice"><strong>Synthetic benchmark.</strong> Every bundled entity, geography, measurement, and source is synthetic benchmark data. No record is a current climate fact.</p><div class="cards"><div class="card"><div>Tasks</div><div class="value">6</div></div><div class="card"><div>Pass rate</div><div class="value">16.7%</div></div><div class="card"><div>Mean score</div><div class="value">0.767</div></div></div>
<div class="tables"><section><h2>Components</h2><table><tr><td>numeric</td><td>66.7%</td></tr><tr><td>unit</td><td>83.3%</td></tr><tr><td>year</td><td>83.3%</td></tr><tr><td>geography</td><td>83.3%</td></tr><tr><td>entity</td><td>83.3%</td></tr><tr><td>citation</td><td>83.3%</td></tr><tr><td>temporal_validity</td><td>66.7%</td></tr></table></section><section><h2>Tasks</h2><table><thead><tr><th>Task</th><th>Score</th><th>Status</th><th>Failures</th></tr></thead><tbody><tr><td>unit-conversion-pass</td><td>1.00</td><td>PASS</td><td>—</td></tr><tr><td>tolerance-fail</td><td>0.70</td><td>FAIL</td><td>numeric_out_of_tolerance</td></tr><tr><td>dimension-fail</td><td>0.60</td><td>FAIL</td><td>unit_dimension_mismatch</td></tr><tr><td>context-fail</td><td>0.70</td><td>FAIL</td><td>year_mismatch, geography_mismatch, entity_mismatch</td></tr><tr><td>source-fail</td><td>0.70</td><td>FAIL</td><td>citation_source_mismatch</td></tr><tr><td>temporal-fail</td><td>0.90</td><td>FAIL</td><td>citation_stale</td></tr></tbody></table></section></div>
<details><summary>Embedded versioned JSON artifact</summary><code>{"artifact_type": "climate-evidence-bench.evaluation-report", "component_weights": {"citation": 0.2, "entity": 0.1, "geography": 0.1, "numeric": 0.3, "temporal_validity": 0.1, "unit": 0.1, "year": 0.1}, "failure_taxonomy": {"citation_future_dated": "The selected source was published after the evaluation as-of date.", "citation_not_yet_valid": "The selected source was not yet valid on the as-of date.", "citation_source_mismatch": "No cited source is accepted for this task.", "citation_stale": "The selected source had expired before the as-of date.", "entity_mismatch": "The answer entity differs after conservative normalization.", "geography_mismatch": "The answer geography differs after conservative normalization.", "invalid_numeric": "The answer value is absent, non-numeric, or non-finite.", "malformed_record": "A task or answer is missing a required field or has an invalid type.", "missing_answer": "No answer record was supplied for the task.", "missing_citation": "The answer includes no citation.", "numeric_out_of_tolerance": "The converted answer is outside absolute and relative tolerance.", "unit_dimension_mismatch": "Answer and expected units describe different dimensions.", "unsupported_unit": "The answer or expected unit is outside the explicit registry.", "year_mismatch": "The answer year differs from the requested entity-year context."}, "generator": {"name": "climate-evidence-bench", "version": "0.1.1"}, "metrics": {"component_pass_rates": {"citation": 0.833333, "entity": 0.833333, "geography": 0.833333, "numeric": 0.666667, "temporal_validity": 0.666667, "unit": 0.833333, "year": 0.833333}, "duplicate_answer_ids": [], "failure_counts": {"citation_source_mismatch": 1, "citation_stale": 1, "entity_mismatch": 1, "geography_mismatch": 1, "numeric_out_of_tolerance": 1, "unit_dimension_mismatch": 1, "year_mismatch": 1}, "mean_score": 0.766667, "orphan_answer_ids": [], "pass_rate": 0.166667, "passed_count": 1, "task_count": 6}, "schema_version": "1.0.0", "synthetic_notice": "Every bundled entity, geography, measurement, and source is synthetic benchmark data. No record is a current climate fact.", "tasks": [{"components": {"citation": true, "entity": true, "geography": true, "numeric": true, "temporal_validity": true, "unit": true, "year": true}, "converted_value_in_expected_unit": 2.4, "failures": [], "passed": true, "score": 1.0, "task_id": "unit-conversion-pass"}, {"components": {"citation": true, "entity": true, "geography": true, "numeric": false, "temporal_validity": true, "unit": true, "year": true}, "converted_value_in_expected_unit": 900.0, "failures": [{"code": "numeric_out_of_tolerance", "message": "Converted value 900 differs from 850; allowed 2."}], "passed": false, "score": 0.7, "task_id": "tolerance-fail"}, {"components": {"citation": true, "entity": true, "geography": true, "numeric": false, "temporal_validity": true, "unit": false, "year": true}, "converted_value_in_expected_unit": null, "failures": [{"code": "unit_dimension_mismatch", "message": "cannot convert GWh (energy) to ktCO2e (emissions)"}], "passed": false, "score": 0.6, "task_id": "dimension-fail"}, {"components": {"citation": true, "entity": false, "geography": false, "numeric": true, "temporal_validity": true, "unit": true, "year": false}, "converted_value_in_expected_unit": 0.42, "failures": [{"code": "year_mismatch", "message": "The answer year differs from the requested entity-year context."}, {"code": "geography_mismatch", "message": "The answer geography differs after conservative normalization."}, {"code": "entity_mismatch", "message": "The answer entity differs after conservative normalization."}], "passed": false, "score": 0.7, "task_id": "context-fail"}, {"components": {"citation": false, "entity": true, "geography": true, "numeric": true, "temporal_validity": false, "unit": true, "year": true}, "converted_value_in_expected_unit": 1250.0, "failures": [{"code": "citation_source_mismatch", "message": "No cited source is accepted for this task."}], "passed": false, "score": 0.7, "task_id": "source-fail"}, {"components": {"citation": true, "entity": true, "geography": true, "numeric": true, "temporal_validity": false, "unit": true, "year": true}, "converted_value_in_expected_unit": 4000.0, "failures": [{"code": "citation_stale", "message": "The selected source had expired before the as-of date."}], "passed": false, "score": 0.9, "task_id": "temporal-fail"}]}</code></details></body></html>