Apply Ruff to yourbench_space/

yourbench_space/leaderboard_space/app.py

@@ -1,7 +1,8 @@
+from env import TASK
+from utils import run_pipeline, update_examples
+
 import gradio as gr
 
-from utils import run_pipeline, update_examples
-from env import TASK
 
 with gr.Blocks(
     title="YourBench Leaderboard",
@@ -11,12 +12,7 @@ with gr.Blocks(
     # DISPLAY TABLE AND ANALYSIS
     title = gr.Markdown(f"YourBench auto-Leaderboard for {TASK}")
     leaderboard = gr.DataFrame(label="Results", interactive=False)
-    samples_ix = gr.Number(
-            label="Example Index", 
-            value=0,
-            step=1,
-            info="Navigate through different examples"
-        )
+    samples_ix = gr.Number(label="Example Index", value=0, step=1, info="Navigate through different examples")
     with gr.Tab("Hardest samples"):
         hard_samples = gr.HTML()
     with gr.Tab("Easiest samples"):
@@ -28,4 +24,4 @@ with gr.Blocks(
 
     demo.load(run_pipeline, [samples_ix], [leaderboard, easy_samples, hard_samples, all_samples])
 
-demo.launch()
+demo.launch()

yourbench_space/leaderboard_space/env.py

@@ -1,4 +1,6 @@
 import os
+
+
 INIT_MODELS = [
     # 70B
     ("Qwen/Qwen2.5-72B-Instruct", "sambanova"),
@@ -7,10 +9,10 @@ INIT_MODELS = [
     # 20 to 30B
     ("Qwen/QwQ-32B", "sambanova"),
     ("mistralai/Mistral-Small-24B-Instruct-2501", "together"),
-    #("allenai/OLMo-2-0325-32B-Instruct", "hf-inference")
-] 
+    # ("allenai/OLMo-2-0325-32B-Instruct", "hf-inference")
+]
 MODELS = [m[0] for m in INIT_MODELS]
 TASK = os.getenv("TASK")
 # With storage
-HF_TOKEN=os.getenv("HF_TOKEN")
+HF_TOKEN = os.getenv("HF_TOKEN")
 ORG_NAME = os.getenv("ORG_NAME")

yourbench_space/leaderboard_space/utils.py

@@ -1,14 +1,14 @@
-from datasets import load_dataset, Dataset
-from functools import lru_cache
+import json
 from typing import Tuple
+
+from env import TASK, MODELS, ORG_NAME
+
 import gradio as gr
-import json
+from datasets import Dataset, load_dataset
 
-from env import MODELS, TASK, ORG_NAME
 
 def aggregate_results() -> list:
-    """From the path of outputs and model list, extracts the current scores and stores them in a list of dicts with model, score, time as keys 
-    """
+    """From the path of outputs and model list, extracts the current scores and stores them in a list of dicts with model, score, time as keys"""
     all_results = []
     for org_model in MODELS:
         try:
@@ -16,16 +16,12 @@ def aggregate_results() -> list:
             ds = load_dataset(path, "results", split="latest")
             config = json.loads(ds["config_general"][0])
             results = json.loads(ds["results"][0])
-                
+
             # Model data
             org, model = org_model.split("/")
 
-            cur_result = {
-                "Org": org, 
-                "Model": model, 
-                "Duration (s)": config["end_time"] - config["start_time"]
-            }
-                    
+            cur_result = {"Org": org, "Model": model, "Duration (s)": config["end_time"] - config["start_time"]}
+
             # Extract the task from the JSON data
             for k_metric, v_dict in results.items():
                 if k_metric != "all":
@@ -36,9 +32,9 @@ def aggregate_results() -> list:
             print(f"Error processing {model} {ORG_NAME}: {e}")
     return all_results
 
+
 def extract_dataviz() -> Tuple[list, list]:
-    """From the path of outputs and model list, extracts from the details the worst samples, best samples
-    """
+    """From the path of outputs and model list, extracts from the details the worst samples, best samples"""
     all_samples = {}
     for org_model in MODELS:
         try:
@@ -51,7 +47,6 @@ def extract_dataviz() -> Tuple[list, list]:
                 score = list(row["metrics"].values())[0]
                 prediction = row["predictions"][0]
 
-
                 # We store flattened samples in a dict
                 # ix -> ix, prompt, gold, model_score for each model, model_prediction for each model
                 # then 2 lists: model_scores and models, to aggreg more easily
@@ -62,7 +57,7 @@ def extract_dataviz() -> Tuple[list, list]:
                         "gold": gold[0] if isinstance(gold, list) else gold,
                         # A bit redundant, but put in their own boxes for simplicity of access later
                         "model_scores": [],
-                        "models": []
+                        "models": [],
                     }
                 if org_model not in all_samples[ix]["models"]:
                     all_samples[ix][f"{org_model}_score"] = row["metrics"]
@@ -73,14 +68,20 @@ def extract_dataviz() -> Tuple[list, list]:
         except Exception as e:
             print(f"Error processing {org_model}: {e}")
 
-    full_samples = sorted(list(all_samples.values()), key= lambda r: r['ix'])
-    hard_samples = sorted([sample for sample in all_samples.values() if sum(sample["model_scores"]) == 0], key= lambda r: r['ix'])
-    easy_samples = sorted([sample for sample in all_samples.values() if sum(sample["model_scores"]) == len(sample["model_scores"])], key= lambda r: r['ix'])
+    full_samples = sorted(all_samples.values(), key=lambda r: r["ix"])
+
+    hard_samples = sorted(
+        [sample for sample in all_samples.values() if sum(sample["model_scores"]) == 0], key=lambda r: r["ix"]
+    )
+    easy_samples = sorted(
+        [sample for sample in all_samples.values() if sum(sample["model_scores"]) == len(sample["model_scores"])],
+        key=lambda r: r["ix"],
+    )
     return easy_samples, hard_samples, full_samples
 
+
 def samples_to_box_display(samples: list, example_index: int = 0):
-    """Adapted from Nathan's code in https://huggingface.co/spaces/SaylorTwift/OpenEvalsModelDetails/
-    """
+    """Adapted from Nathan's code in https://huggingface.co/spaces/SaylorTwift/OpenEvalsModelDetails/"""
     if len(samples) == 0:
         return "No samples in this category!"
     outputs = []
@@ -88,21 +89,21 @@ def samples_to_box_display(samples: list, example_index: int = 0):
     for model in sample["models"]:
         try:
             outputs.append({
-                'Model': model,
-                'Prediction': sample[f'{model}_prediction'],
-                'Prompt': sample['prompt'],
-                'Metrics': sample[f'{model}_score'],
-                'Gold': sample['gold']
+                "Model": model,
+                "Prediction": sample[f"{model}_prediction"],
+                "Prompt": sample["prompt"],
+                "Metrics": sample[f"{model}_score"],
+                "Gold": sample["gold"],
             })
         except (KeyError, IndexError):
             continue
-    
+
     if not outputs:
         return "No results found for the selected combination."
-    
+
     # Create HTML output with all models
     html_output = "<div style='max-width: 800px; margin: 0 auto;'>\n\n"
-    
+
     # Show gold answer at the top with distinct styling
     if outputs:
         html_output += "<div style='background: #e6f3e6; padding: 20px; border-radius: 10px; margin-bottom: 20px;'>\n"
@@ -111,15 +112,15 @@ def samples_to_box_display(samples: list, example_index: int = 0):
         html_output += f"<pre style='white-space: pre-wrap; word-wrap: break-word; margin: 0;'><code>{outputs[0]['Gold']}</code></pre>\n"
         html_output += "</div>\n"
         html_output += "</div>\n"
-    
+
     for output in outputs:
         html_output += "<div style='background: #f5f5f5; padding: 20px; margin-bottom: 20px; border-radius: 10px;'>\n"
         html_output += f"<h2 style='margin-top: 0;'>{output['Model']}</h2>\n"
-        
+
         # Format metrics as a clean table
         html_output += "<details open style='margin-bottom: 15px;'>\n"
         html_output += "<summary><h3 style='display: inline; margin: 0;'>Metrics</h3></summary>\n"
-        metrics = output['Metrics']
+        metrics = output["Metrics"]
         if isinstance(metrics, str):
             metrics = eval(metrics)
         html_output += "<div style='overflow-x: auto;'>\n"
@@ -131,17 +132,17 @@ def samples_to_box_display(samples: list, example_index: int = 0):
         html_output += "</table>\n"
         html_output += "</div>\n"
         html_output += "</details>\n\n"
-        
+
         # Handle prompt formatting with better styling
         html_output += "<details style='margin-bottom: 15px;'>\n"
         html_output += "<summary><h3 style='display: inline; margin: 0;'>Prompt</h3></summary>\n"
         html_output += "<div style='background: #ffffff; padding: 15px; border-radius: 5px; margin-top: 10px;'>\n"
-        
-        prompt_text = output['Prompt']
+
+        prompt_text = output["Prompt"]
         if isinstance(prompt_text, list):
             for i, msg in enumerate(prompt_text):
-                if isinstance(msg, dict) and 'content' in msg:
-                    role = msg.get('role', 'message').title()
+                if isinstance(msg, dict) and "content" in msg:
+                    role = msg.get("role", "message").title()
                     html_output += "<div style='margin-bottom: 10px;'>\n"
                     html_output += f"<strong>{role}:</strong>\n"
                     html_output += "<div style='overflow-x: auto;'>\n"
@@ -156,20 +157,20 @@ def samples_to_box_display(samples: list, example_index: int = 0):
                     html_output += "</div>\n"
         else:
             html_output += "<div style='overflow-x: auto;'>\n"
-            if isinstance(prompt_text, dict) and 'content' in prompt_text:
+            if isinstance(prompt_text, dict) and "content" in prompt_text:
                 html_output += f"<pre style='white-space: pre-wrap; word-wrap: break-word; margin: 5px 0;'><code>{prompt_text['content']}</code></pre>\n"
             else:
                 html_output += f"<pre style='white-space: pre-wrap; word-wrap: break-word; margin: 5px 0;'><code>{prompt_text}</code></pre>\n"
             html_output += "</div>\n"
-        
+
         html_output += "</div>\n"
         html_output += "</details>\n\n"
-        
+
         # Style prediction output - now in a collapsible section
         html_output += "<details open style='margin-bottom: 15px;'>\n"
         html_output += "<summary><h3 style='display: inline; margin: 0;'>Prediction</h3>"
         # Add word count in a muted style
-        word_count = len(output['Prediction'].split())
+        word_count = len(output["Prediction"].split())
         html_output += f"<span style='color: #666; font-size: 0.8em; margin-left: 10px;'>({word_count} words)</span>"
         html_output += "</summary>\n"
         html_output += "<div style='background: #ffffff; padding: 15px; border-radius: 5px; margin-top: 10px;'>\n"
@@ -179,20 +180,30 @@ def samples_to_box_display(samples: list, example_index: int = 0):
         html_output += "</div>\n"
         html_output += "</details>\n"
         html_output += "</div>\n\n"
-    
+
     html_output += "</div>"
     return html_output
 
+
 def run_pipeline(samples_ix: int = 0):
     results = aggregate_results()
     best_samples, worst_samples, all_samples = extract_dataviz()
-    return gr.Dataframe(Dataset.from_list(results).to_pandas(), visible=True), \
-        gr.HTML(samples_to_box_display(best_samples, samples_ix), label="Easiest samples (always found)", visible=True), \
-        gr.HTML(samples_to_box_display(worst_samples, samples_ix), label="Hardest samples (always failed)", visible=True), \
-        gr.HTML(samples_to_box_display(all_samples, samples_ix), label="All samples", visible=True)
+    return (
+        gr.Dataframe(Dataset.from_list(results).to_pandas(), visible=True),
+        gr.HTML(
+            samples_to_box_display(best_samples, samples_ix), label="Easiest samples (always found)", visible=True
+        ),
+        gr.HTML(
+            samples_to_box_display(worst_samples, samples_ix), label="Hardest samples (always failed)", visible=True
+        ),
+        gr.HTML(samples_to_box_display(all_samples, samples_ix), label="All samples", visible=True),
+    )
+
 
 def update_examples(samples_ix: int = 0):
     best_samples, worst_samples, all_samples = extract_dataviz()
-    return samples_to_box_display(best_samples, samples_ix), \
-        samples_to_box_display(worst_samples, samples_ix), \
-        samples_to_box_display(all_samples, samples_ix)
+    return (
+        samples_to_box_display(best_samples, samples_ix),
+        samples_to_box_display(worst_samples, samples_ix),
+        samples_to_box_display(all_samples, samples_ix),
+    )