add leaderboard

app.py

@@ -0,0 +1,107 @@
+import gradio as gr
+from gradio_leaderboard import Leaderboard
+from pathlib import Path
+import pandas as pd
+
+import os
+
+import json 
+
+from envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
+
+
+def submit(model_name, model_id, challenge, submission_id, architecture, license):
+    
+    if model_name == "" or model_id == "" or challenge == "" or submission_id == "" or architecture == "" or license == "":
+        gr.Error("Please fill all the fields")
+        return
+    try:
+        user_name = ""
+        if "/" in model_id:
+            user_name = model_id.split("/")[0]
+            model_path = model_id.split("/")[1]
+
+        eval_entry = {
+            "model_name": model_name,
+            "model_id": model_id,
+            "challenge": challenge,
+            "submission_id": submission_id,
+            "architecture": architecture,
+            "license": license
+        }
+        OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
+        os.makedirs(OUT_DIR, exist_ok=True)
+        out_path = f"{OUT_DIR}/{user_name}_{model_path}.json"
+
+        with open(out_path, "w") as f:
+            f.write(json.dumps(eval_entry))
+
+        print("Uploading eval file")
+        API.upload_file(
+            path_or_fileobj=out_path,
+            path_in_repo=out_path.split("eval-queue/")[1],
+            repo_id=QUEUE_REPO,
+            repo_type="dataset",
+            commit_message=f"Add {model_name} to eval queue",
+        )
+        gr.Info("Successfully submitted", duration=10)
+        # Remove the local file
+        os.remove(out_path)
+    except:
+        gr.Error("Error submitting the model")
+
+
+
+
+
+abs_path = Path(__file__).parent
+
+# Any pandas-compatible data
+df = pd.read_json(str(abs_path / "leaderboard_data.json"))
+
+with gr.Blocks() as demo:
+    gr.Markdown("""
+    # MLSB 2024 Challenges
+    """)
+
+
+    with gr.Tab("🎖️ PINDER Leaderboard"):
+        gr.Markdown("""## PINDER Leaderboard
+                Evaluating Protein-Protein interaction prediction
+                """)
+        Leaderboard(
+        value=df,
+        select_columns=["Arch", "Model", "L_rms", "I_rms",
+            "F_nat", "DOCKQ", "CAPRI"],
+        search_columns=["model_name_for_query"],
+        hide_columns=["model_name_for_query",],
+        filter_columns=["Arch"],
+    )
+    with gr.Tab("🥇 PLINDER Leaderboard"):
+        gr.Markdown("""## PLINDER Leaderboard
+                Evaluating Protein-Ligand prediction
+                """)
+        Leaderboard(
+        value=df,
+        select_columns=["Arch", "Model", "L_rms", "I_rms",
+            "F_nat", "DOCKQ", "CAPRI"],
+        search_columns=["model_name_for_query"],
+        hide_columns=["model_name_for_query",],
+        filter_columns=["Arch"],
+    )
+    with gr.Tab("✉️ Submit"):
+        gr.Markdown("""## Submit your model
+                Submit your model to the leaderboard
+                """)
+        model_name = gr.Textbox(label="Model name")
+        model_id = gr.Textbox(label="username/space e.g mlsb/alphafold3")
+        challenge = gr.Radio(choices=["PINDER", "PLINDER"],label="Challenge")
+        submission_id = gr.Textbox(label="Submission ID on CMT")
+        architecture = gr.Dropdown(choices=["GNN", "CNN", "Physics-based", "Other"],label="Model architecture")
+        license = gr.Dropdown(choices=["mit", "apache-2.0", "gplv2", "gplv3", "lgpl", "mozilla", "bsd", "other"],label="License")
+        submit_btn = gr.Button("Submit")
+
+        submit_btn.click(submit, inputs=[model_name, model_id, challenge, submission_id, architecture, license], outputs=[])
+
+if __name__ == "__main__":
+    demo.launch()

envs.py

@@ -0,0 +1,25 @@
+import os
+
+from huggingface_hub import HfApi
+
+# Info to change for your repository
+# ----------------------------------
+TOKEN = os.environ.get("HF_TOKEN") # A read/write token for your org
+
+OWNER = "MLSB" # Change to your org - don't forget to create a results and request dataset, with the correct format!
+# ----------------------------------
+
+REPO_ID = f"{OWNER}/leaderboard2024"
+QUEUE_REPO = f"{OWNER}/requests"
+RESULTS_REPO = f"{OWNER}/results"
+
+# If you setup a cache later, just change HF_HOME
+CACHE_PATH=os.getenv("HF_HOME", ".")
+
+# Local caches
+EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
+EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
+EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
+EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
+
+API = HfApi(token=TOKEN)

leaderboard_data.json

@@ -0,0 +1 @@
+{"Arch":{"0":"GNN"},"Model":{"0":"davidkim205/Rhea-72b-v0.5"},"L_rms":{"0":81.22},"I_rms":{"0":79.78},"F_nat":{"0":91.15},"DOCKQ":{"0":77.95},"CAPRI":{"0":74.5},"Runtime":{"0":"2 +-0.2"},"Hub License":{"0":"apache-2.0"},"#Params (B)":{"0":72.29},"Model sha":{"0":"fda5cf998a0f2d89b53b5fa490793e3e50bb8239"},"model_name_for_query":{"0":"davidkim205\/Rhea-72b-v0.5"}}