Filter think cases

ACKNOWLEDGMENTS.md

@@ -0,0 +1,7 @@
+## Acknowledgments
+
+This work builds upon:
+- EleutherAI/lm-evaluation-harness
+- eduagarcia/lm-evaluation-harness-pt
+
+We thank the authors and maintainers of both repositories for making their work openly available.

README.md

@@ -1,3 +1,26 @@
+# Language Model Evaluation Harness for Portuguese benchmark and Steered
+
+## Provenance and Adaptations
+
+This project is based on the original `lm-evaluation-harness` from EleutherAI.
+
+To support Portuguese evaluation tasks together with steering-based evaluation, selected components and task definitions were adapted/ported from the fork:
+
+- `eduagarcia/lm-evaluation-harness-pt`
+
+The current repository keeps the upstream EleutherAI harness as the main evaluation engine and incorporates Portuguese task support on top of it.
+
+Main motivation:
+- preserve compatibility with the current `steered` backend
+- reuse valuable Portuguese evaluation tasks and utilities
+- enable steering experiments on Portuguese benchmarks
+
+This repository does not claim authorship of the original harness or the original Portuguese task implementations; 
+it integrates and adapts them for the present research workflow.
+
+See [eduagarcia/lm-evalution-harness-pt](https://github.com/eduagarcia/lm-evaluation-harness-pt)
+
+
 # Language Model Evaluation Harness
 
 [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.10256836.svg)](https://doi.org/10.5281/zenodo.10256836)

lm_eval/api/metrics.py

@@ -4,6 +4,8 @@
 import random
 import re
 import string
+import sklearn.metrics
+import scipy.stats
 from collections.abc import Iterable
 from typing import Callable, List, Optional, Sequence, TypeVar
 
@@ -148,6 +150,23 @@ def brier_score(items):  # This is a passthrough function
 def brier_score_fn(items):  # This is a passthrough function
     return items
 
+@register_aggregation("f1_macro_score")
+def f1_macro_score(items):
+    unzipped_list = list(zip(*items))
+    golds = unzipped_list[0]
+    preds = unzipped_list[1]
+    fscore = sklearn.metrics.f1_score(golds, preds, labels=list(set(golds)), average='macro')
+
+    return fscore
+
+@register_aggregation("pt_acc")
+def pt_acc_score(items):
+    unzipped_list = list(zip(*items))
+    golds = unzipped_list[0]
+    preds = unzipped_list[1]
+    acc_score = sklearn.metrics.accuracy_score(golds, preds)
+
+    return acc_score
 
 @register_metric(
     metric="acc",
@@ -338,6 +357,43 @@ def bypass(items):
 def mcc_fn(items):  # This is a passthrough function
     return items
 
+@register_aggregation("pearsonr")
+def pearsonr(items):
+    unzipped_list = list(zip(*items))
+    golds = unzipped_list[0]
+    preds = unzipped_list[1]
+
+    res = scipy.stats.pearsonr(golds, preds)[0]
+    if np.isnan(res):
+        return 0
+
+    return abs(res)
+
+@register_aggregation("mean_squared_error")
+def mean_squared_error(items):
+    unzipped_list = list(zip(*items))
+    golds = unzipped_list[0]
+    preds = unzipped_list[1]
+    # print(preds)
+    return sklearn.metrics.mean_squared_error(golds, preds)
+
+@register_metric(
+    metric="pearson",
+    higher_is_better=True,
+    output_type="multiple_choice",
+    aggregation="pearsonr",
+)
+def pearson_fn(items):  # This is a passthrough function
+    return items
+
+@register_metric(
+    metric="mse",
+    higher_is_better=False,
+    output_type="multiple_choice",
+    aggregation="mean_squared_error",
+)
+def mse_fn(items):  # This is a passthrough function
+    return items
 
 @register_metric(
     metric="f1",
@@ -348,6 +404,14 @@ def mcc_fn(items):  # This is a passthrough function
 def f1_fn(items):  # This is a passthrough function
     return items
 
+@register_metric(
+    metric="f1_macro_score",
+    higher_is_better=True,
+    output_type="multiple_choice",
+    aggregation="f1_macro_score",
+)
+def f1_macro_fn(items):  # This is a passthrough function
+    return items
 
 @register_metric(
     metric="bleu",
@@ -571,7 +635,11 @@ def stderr_for_metric(
     bootstrappable = [
         median,
         matthews_corrcoef,
+        pearsonr,
+        mse_fn,
         f1_score,
+        f1_macro_score,
+        pt_acc_score,
         perplexity,
         bleu,
         chrf,

lm_eval/filters/__init__.py

@@ -5,7 +5,7 @@
 from lm_eval.api.filter import FilterEnsemble
 from lm_eval.api.registry import filter_registry, get_filter
 
-from . import custom, extraction, selection, transformation
+from . import custom, decontamination, extraction, selection, transformation
 
 
 def build_filter_ensemble(
@@ -26,6 +26,7 @@ def build_filter_ensemble(
 
 __all__ = [
     "custom",
+    "decontamination",
     "extraction",
     "selection",
     "transformation",