Fix bug and Add Avg column

Author: speed1313

github_pages/public/leaderboard.json

@@ -37,6 +37,9 @@
       },
       "JMMMU": {
         "Acc": 0.26
+      },
+      "Avg": {
+        "Avg": 41.98
       }
     }
   },
@@ -78,6 +81,9 @@
       },
       "JMMMU": {
         "Acc": 0.03
+      },
+      "Avg": {
+        "Avg": 45.66
       }
     }
   },
@@ -119,6 +125,9 @@
       },
       "JMMMU": {
         "Acc": 0.36
+      },
+      "Avg": {
+        "Avg": 66.84
       }
     }
   },
@@ -160,6 +169,9 @@
       },
       "JMMMU": {
         "Acc": 0.06
+      },
+      "Avg": {
+        "Avg": 48.99
       }
     }
   },
@@ -201,6 +213,9 @@
       },
       "JMMMU": {
         "Acc": 0.19
+      },
+      "Avg": {
+        "Avg": 71.43
       }
     }
   },
@@ -242,6 +257,9 @@
       },
       "JMMMU": {
         "Acc": 0.39
+      },
+      "Avg": {
+        "Avg": 67.55
       }
     }
   },
@@ -283,6 +301,9 @@
       },
       "JMMMU": {
         "Acc": 0.43
+      },
+      "Avg": {
+        "Avg": 69.84
       }
     }
   },
@@ -324,6 +345,9 @@
       },
       "JMMMU": {
         "Acc": 0.22
+      },
+      "Avg": {
+        "Avg": 40.0
       }
     }
   },
@@ -365,6 +389,9 @@
       },
       "JMMMU": {
         "Acc": 0.3
+      },
+      "Avg": {
+        "Avg": 58.31
       }
     }
   },
@@ -406,6 +433,9 @@
       },
       "JMMMU": {
         "Acc": 0.25
+      },
+      "Avg": {
+        "Avg": 53.61
       }
     }
   },
@@ -447,6 +477,9 @@
       },
       "JMMMU": {
         "Acc": 0.37
+      },
+      "Avg": {
+        "Avg": 74.26
       }
     }
   },
@@ -488,6 +521,9 @@
       },
       "JMMMU": {
         "Acc": 0.19
+      },
+      "Avg": {
+        "Avg": 67.41
       }
     }
   },
@@ -529,6 +565,9 @@
       },
       "JMMMU": {
         "Acc": 0.35
+      },
+      "Avg": {
+        "Avg": 62.73
       }
     }
   },
@@ -543,6 +582,9 @@
         "LLM": 3.23,
         "Rouge": 12.97
       },
+      "JIC": {
+        "Acc": 0.58
+      },
       "MECHA": {
         "Acc": 0.46
       },
@@ -567,6 +609,9 @@
       },
       "JMMMU": {
         "Acc": 0.33
+      },
+      "Avg": {
+        "Avg": 64.57
       }
     }
   },
@@ -608,6 +653,9 @@
       },
       "JMMMU": {
         "Acc": 0.39
+      },
+      "Avg": {
+        "Avg": 66.59
       }
     }
   },
@@ -649,6 +697,9 @@
       },
       "JMMMU": {
         "Acc": 0.39
+      },
+      "Avg": {
+        "Avg": 67.57
       }
     }
   },
@@ -690,6 +741,9 @@
       },
       "JMMMU": {
         "Acc": 0.48
+      },
+      "Avg": {
+        "Avg": 78.59
       }
     }
   },
@@ -731,6 +785,9 @@
       },
       "JMMMU": {
         "Acc": 0.61
+      },
+      "Avg": {
+        "Avg": 87.61
       }
     }
   },
@@ -772,6 +829,9 @@
       },
       "JMMMU": {
         "Acc": 0.37
+      },
+      "Avg": {
+        "Avg": 68.08
       }
     }
   },
@@ -813,6 +873,9 @@
       },
       "JMMMU": {
         "Acc": 0.48
+      },
+      "Avg": {
+        "Avg": 78.04
       }
     }
   },
@@ -854,6 +917,9 @@
       },
       "JMMMU": {
         "Acc": 0.51
+      },
+      "Avg": {
+        "Avg": 78.5
       }
     }
   },
@@ -895,6 +961,9 @@
       },
       "JMMMU": {
         "Acc": 0.39
+      },
+      "Avg": {
+        "Avg": 67.33
       }
     }
   },
@@ -936,6 +1005,9 @@
       },
       "JMMMU": {
         "Acc": 0.57
+      },
+      "Avg": {
+        "Avg": 87.94
       }
     }
   }

scripts/make_leaderboard.py

@@ -75,6 +75,12 @@ def process_results(result_dir: str, model_list: List[str]) -> pd.DataFrame:
             for k in df.columns
         }
     )
+    # すべてのスコアを 100 点満点に正規化
+    df_normalized = df.apply(lambda x: x / x.max() * 100, axis=0)
+
+    # 各モデルの全体スコア（平均）を計算し、最後の列に追加
+    df["Avg/Avg"] = df_normalized.mean(axis=1).round(2)
+
     return df
 
 
@@ -93,6 +99,9 @@ def generate_json_path(df: pd.DataFrame, output_path: str):
 
         for col, score in row.items():
             if isinstance(score, (int, float)) and not pd.isna(score):
+                if "/" not in col:
+                    model_entry["scores"][col] = score
+                    continue
                 task, metric = col.split("/")
                 if task not in model_entry["scores"]:
                     model_entry["scores"][task] = {}
@@ -145,6 +154,23 @@ def plot_task_performance(df: pd.DataFrame):
 
 def format_output(df: pd.DataFrame, output_format: str) -> str:
     """Format the DataFrame output for markdown or LaTeX."""
+
+    # textbf top1 score and underline top2 score for each task
+    for col in df.columns:
+        top1_model = df[col].astype(float).idxmax()
+        top2_model = df[col].astype(float).nlargest(2).index[-1]
+        top1_score = f"{float(df.loc[top1_model, col]):.3g}"
+        top2_score = f"{float(df.loc[top2_model, col]):.3g}"
+        # apply formatting
+        if output_format == "latex":
+            df.loc[top1_model, col] = f"\\textbf{{{top1_score}}}"
+            df.loc[top2_model, col] = f"\\textit{{{top2_score}}}"
+            df.loc[top2_model, col] = f"\\underline{{{top2_score}}}"
+        else:
+            df.loc[top1_model, col] = f"**{top1_score}**"
+            df.loc[top2_model, col] = f"*{top2_score}*"
+            df.loc[top2_model, col] = f"<u>{top2_score}</u>"
+
     df = df.fillna("")
     if output_format == "markdown":
         return df.to_markdown(mode="github", floatfmt=".3g")
@@ -164,20 +190,20 @@ def main(
 ):
     df = process_results(result_dir, model_list)
     if plot_corr:
-        plot_correlation(df, "correlation.png")
+        plot_correlation(df.copy(), "correlation.png")
     # plot_correlation(df.T, "correlation_model.png")
     if plot_bar:
-        plot_task_performance(df)
+        plot_task_performance(df.copy())
 
-    table = format_output(df, output_format)
+    table = format_output(df.copy(), output_format)
     print(table)
 
     if output_path:
         with open(output_path, "w") as f:
             f.write(table)
 
     if update_pages:
-        generate_json_path(df, "github_pages/public/leaderboard.json")
+        generate_json_path(df.copy(), "github_pages/public/leaderboard.json")
 
 
 def parse_args():