fix: add charts

app.py

@@ -186,9 +186,25 @@ with demo:
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
         with gr.TabItem("🏆 U-MATH", elem_id="u-math-benchmark-tab-table", id=0):
             leaderboard_umath = init_leaderboard(LEADERBOARD_U_MATH_DF, U_MATH_COLUMNS_DICT)
+            gr.ScatterPlot(
+                value=LEADERBOARD_U_MATH_DF,
+                title="U-MATH: Text vs Visual Accuracy",
+                x=U_MATH_COLUMNS_DICT["u_math_text_acc"].pretty_name,
+                y=U_MATH_COLUMNS_DICT["u_math_visual_acc"].pretty_name,
+                color=U_MATH_COLUMNS_DICT["model_family"].pretty_name,
+                tooltip=[U_MATH_COLUMNS_DICT["full_model_name"].pretty_name, U_MATH_COLUMNS_DICT["u_math_acc"].pretty_name],
+            )
 
         with gr.TabItem("🏅 μ-MATH (Meta-Benchmark)", elem_id="mu-math-benchmark-tab-table", id=1):
             leaderboard_mumath = init_leaderboard(LEADERBOARD_MU_MATH_DF, MU_MATH_COLUMNS_DICT)
+            gr.ScatterPlot(
+                value=LEADERBOARD_MU_MATH_DF,
+                title="μ-MATH: True Positive Rate (Recall) vs True Negative Rate (Specificity)",
+                x=MU_MATH_COLUMNS_DICT["mu_math_tpr"].pretty_name,
+                y=MU_MATH_COLUMNS_DICT["mu_math_tnr"].pretty_name,
+                color=MU_MATH_COLUMNS_DICT["model_family"].pretty_name,
+                tooltip=[MU_MATH_COLUMNS_DICT["full_model_name"].pretty_name, MU_MATH_COLUMNS_DICT["mu_math_f1"].pretty_name],
+            )
 
         with gr.TabItem("📝 About", elem_id="about-tab-table", id=2):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
@@ -205,4 +221,5 @@ with demo:
 scheduler = BackgroundScheduler()
 scheduler.add_job(restart_space, "interval", seconds=60 * 60)
 scheduler.start()
-demo.queue(default_concurrency_limit=40).launch(ssr_mode=False)
+# demo.queue(default_concurrency_limit=40).launch(ssr_mode=False)
+demo.queue(default_concurrency_limit=40).launch()

data/mu_math_eval_results.json

@@ -2,19 +2,19 @@
   {
     "model_name": "mistralai/Ministral-8B-Instruct-2410",
     "extract_model_name": "Qwen/Qwen2.5-72B-Instruct",
-    "mu_math": [0.664, 0.33, 0.651, 0.68, 0.701, 0.628],
-    "GPT-4o": [0.664, 0.332, 0.621, 0.71, 0.696, 0.637],
-    "Gemini-1.5-Pro": [0.672, 0.279, 0.709, 0.585, 0.798, 0.466],
-    "Llama-3.1-70B-Instruct": [0.675, 0.317, 0.619, 0.707, 0.541, 0.769],
-    "Qwen2.5-72B-Instruct": [0.646, 0.295, 0.626, 0.672, 0.719, 0.574]
+    "mu_math": [0.664, 0.33, 0.651, 0.68, 0.701, 0.628, 0.574],
+    "GPT-4o": [0.664, 0.332, 0.621, 0.71, 0.696, 0.637, 0.574],
+    "Gemini-1.5-Pro": [0.672, 0.279, 0.709, 0.585, 0.798, 0.466, 0.574],
+    "Llama-3.1-70B-Instruct": [0.675, 0.317, 0.619, 0.707, 0.541, 0.769, 0.574],
+    "Qwen2.5-72B-Instruct": [0.646, 0.295, 0.626, 0.672, 0.719, 0.574, 0.574]
   },
   {
     "model_name": "meta-llama/Llama-3.3-70B-Instruct",
     "extract_model_name": "Qwen/Qwen2.5-72B-Instruct",
-    "mu_math": [0.741, 0.496, 0.666, 0.827, 0.816, 0.682],
-    "GPT-4o": [0.731, 0.475, 0.636, 0.832, 0.802, 0.681],
-    "Gemini-1.5-Pro": [0.705, 0.394, 0.693, 0.732, 0.856, 0.508],
-    "Llama-3.1-70B-Instruct": [0.823, 0.605, 0.67, 0.908, 0.802, 0.832],
-    "Qwen2.5-72B-Instruct": [0.705, 0.421, 0.658, 0.767, 0.791, 0.627]
+    "mu_math": [0.741, 0.496, 0.666, 0.827, 0.816, 0.682, 0.574],
+    "GPT-4o": [0.731, 0.475, 0.636, 0.832, 0.802, 0.681, 0.574],
+    "Gemini-1.5-Pro": [0.705, 0.394, 0.693, 0.732, 0.856, 0.508, 0.574],
+    "Llama-3.1-70B-Instruct": [0.823, 0.605, 0.67, 0.908, 0.802, 0.832, 0.574],
+    "Qwen2.5-72B-Instruct": [0.705, 0.421, 0.658, 0.767, 0.791, 0.627, 0.574]
   }
 ]

src/populate.py

@@ -7,6 +7,9 @@ from huggingface_hub import model_info
 from transformers import AutoConfig
 
 
+UNKNOWN_MODEL_SHOW_SIZE = 150
+
+
 def is_model_on_hub(
     model_name: str, revision: str, token: str = None, trust_remote_code=False
 ) -> tuple[bool, str | None, str | None]:
@@ -48,6 +51,22 @@ def model_type_to_symbol(model_type: str) -> str:
 
 def get_hf_data_by_model_name(model_name: str) -> dict:
     """Get model data from Hugging Face API by model name"""
+    model_family = "Unknown"
+    if 'mistral' in model_name.lower() or 'numina' in model_name.lower():
+        model_family = "Mistral"
+    elif 'meta-llama' in model_name.lower():
+        model_family = "LLaMA"
+    elif 'claude' in model_name.lower():
+        model_family = "Claude"
+    elif 'qwen' in model_name.lower() or 'athene' in model_name.lower() or 'qwq' in model_name.lower() or 'qvq' in model_name.lower():
+        model_family = "Qwen"
+    elif 'gpt' in model_name.lower() or 'o1' in model_name.lower():
+        model_family = "GPT"
+    elif 'gemini' in model_name.lower():
+        model_family = "Gemini"
+    elif 'deepseek' in model_name.lower():
+        model_family = "DeepSeek"
+
     still_on_hub, _, model_config = is_model_on_hub(model_name, "main", trust_remote_code=True)
     if not still_on_hub and '/' in model_name:
         print(f"Model {model_name} is not on the hub, try unsloth/...")
@@ -69,6 +88,8 @@ def get_hf_data_by_model_name(model_name: str) -> dict:
             print("SafeTensors not found in", model_name, e)
             if 'Pixtral-12B' in model_name:
                 num_params = 12
+            elif 'Pixtral-Large-Instruct-2411' in model_name:
+                num_params = 123.3
             pass
         print("num_params", model_name, num_params)
 
@@ -93,6 +114,7 @@ def get_hf_data_by_model_name(model_name: str) -> dict:
         "model_size": num_params if num_params else None,
         "model_url": model_url,
         "model_license": model_license,
+        "model_family": model_family,
     }
 
 
@@ -109,11 +131,14 @@ class Field:
 MODEL_COLUMNS_DICT = {
     "model_type_symbol": Field("T", "str", never_hidden=True),
     "model_size_symbol": Field("S", "str", never_hidden=True),
+    "full_model_name": Field("Full Model Name", "markdown", fully_hidden=True),
     "model_name": Field("Model Name", "markdown", never_hidden=True),
     "model_type": Field("Type", "str", displayed_by_default=False),
     "model_size": Field("#Params (B)", "number", displayed_by_default=False),
+    "model_size_including_unknown": Field("#Params inc. Proprietary (B)", "number", fully_hidden=True),
     "model_architecture": Field("Architecture", "str", displayed_by_default=False),
     "model_license": Field("License", "markdown", displayed_by_default=False),
+    "model_family": Field("Family", "str", displayed_by_default=False),
 }
 
 U_MATH_COLUMNS_DICT = {
@@ -233,8 +258,11 @@ def get_u_math_leaderboard_df() -> pd.DataFrame:
     df["model_license"] = df["model_name"].apply(lambda x: model_to_meta_dict[x]["model_license"])
     df["model_type"] = df["model_name"].apply(lambda x: model_to_meta_dict[x]["model_type"])
     df["model_type_symbol"] = df["model_type"].apply(model_type_to_symbol)
+    df["model_family"] = df["model_name"].apply(lambda x: model_to_meta_dict[x]["model_family"])
     df["model_size"] = df["model_name"].apply(lambda x: model_to_meta_dict[x]["model_size"])
+    df["model_size_including_unknown"] = df["model_size"].apply(lambda x: x if x and pd.notna(x) else UNKNOWN_MODEL_SHOW_SIZE).astype(float)
     df["model_size_symbol"] = df["model_size"].apply(model_size_to_symbol)
+    df["full_model_name"] = df["model_name"]
     df["model_name"] = df["model_name"].apply(
         lambda x: f"[{x}]({url})" if (url := model_to_meta_dict[x]["model_url"]) else x
     )
@@ -253,12 +281,16 @@ def get_mu_math_leaderboard_df() -> pd.DataFrame:
 
     # Calculate columns with prefixes f1, tpr, tnr, ppv, npv
     for col in ["mu_math", "GPT-4o", "Gemini-1.5-Pro", "Llama-3.1-70B-Instruct", "Qwen2.5-72B-Instruct"]:
-        df[col + "_f1"] = df[col].apply(lambda x: x[0])
-        df[col + "_tpr"] = df[col].apply(lambda x: x[1])
-        df[col + "_tnr"] = df[col].apply(lambda x: x[2])
-        df[col + "_ppv"] = df[col].apply(lambda x: x[3])
-        df[col + "_npv"] = df[col].apply(lambda x: x[4])
+        df[col + "_acc"] = df[col].apply(lambda x: x[0])
+        df[col + "_f1"] = df[col].apply(lambda x: x[1])
+        df[col + "_mcc"] = df[col].apply(lambda x: x[2])
+        df[col + "_tpr"] = df[col].apply(lambda x: x[3])
+        df[col + "_tnr"] = df[col].apply(lambda x: x[4])
+        df[col + "_ppv"] = df[col].apply(lambda x: x[5])
+        df[col + "_npv"] = df[col].apply(lambda x: x[6])
         del df[col]
+        del df[col + "_acc"]
+        del df[col + "_mcc"]
 
     # # flatten list [x, y, z] in columns as ["_f1", "_precision", "_recall"] suffixes for columns
     # for col in ["mu_math", "GPT-4o", "Gemini-1.5-Pro", "Llama-3.1-70B-Instruct", "Qwen2.5-72B-Instruct"]:
@@ -281,8 +313,11 @@ def get_mu_math_leaderboard_df() -> pd.DataFrame:
     df["model_license"] = df["model_name"].apply(lambda x: model_to_meta_dict[x]["model_license"])
     df["model_type"] = df["model_name"].apply(lambda x: model_to_meta_dict[x]["model_type"])
     df["model_type_symbol"] = df["model_type"].apply(model_type_to_symbol)
+    df["model_family"] = df["model_name"].apply(lambda x: model_to_meta_dict[x]["model_family"])
     df["model_size"] = df["model_name"].apply(lambda x: model_to_meta_dict[x]["model_size"])
+    df["model_size_including_unknown"] = df["model_size"].apply(lambda x: x if x and pd.notna(x) else UNKNOWN_MODEL_SHOW_SIZE).astype(float)
     df["model_size_symbol"] = df["model_size"].apply(model_size_to_symbol)
+    df["full_model_name"] = df["model_name"]
     df["model_name"] = df["model_name"].apply(
         lambda x: f"[{x}]({url})" if (url := model_to_meta_dict[x]["model_url"]) else x
     )