Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+assets/Instance[[:space:]]Solve[[:space:]]Count[[:space:]]Distribution.png filter=lfs diff=lfs merge=lfs -text
+assets/Performance[[:space:]]Comparison[[:space:]]Across[[:space:]]Subsets.png filter=lfs diff=lfs merge=lfs -text
+assets/SWE-Bench[[:space:]]Verified[[:space:]]Discriminative[[:space:]]Subsets.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,69 @@
+# Large experimental data directory (will be cloned by automated updater)
+leaderboard_swe_bench_experiments/
+
+# Cache directory (generated by app)
+leaderboard_cache/
+
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Virtual environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# IDEs
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# OS generated files
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db
+
+# Temporary files
+*.tmp
+*.temp
+temp/
+tmp/
+
+# Logs
+*.log
+logs/
+
+# Environment variables
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local

README.md

@@ -1,12 +1,75 @@
 ---
-title: Swe Bench Verified Discriminative Leaderboard
-emoji: 📉
-colorFrom: purple
-colorTo: yellow
+title: SWE-Bench Verified Discriminative Subsets Leaderboard
+emoji: 🏆
+colorFrom: blue
+colorTo: red
 sdk: gradio
 sdk_version: 5.34.2
 app_file: app.py
 pinned: false
+license: mit
+tags:
+- leaderboard
+- software-engineering
+- swe-bench
+- evaluation
+- benchmark
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# 🏆 SWE-Bench Verified Discriminative Subsets Leaderboard
+
+This interactive leaderboard displays SWE-agents performance across SWE-Bench_Verified and four discriminative subsets designed to provide enhanced evaluation sensitivity for state-of-the-art systems.
+
+## 🎯 Why Discriminative Subsets?
+
+As SWE-agents improve, achieving 70%+ success rates on the full SWE-Bench Verified benchmark, traditional evaluation loses discriminative power. These targeted subsets focus on the most challenging problems to better distinguish between top-tier systems.
+
+## 📊 The Four Discriminative Subsets
+
+1. **🔥 Frontier Subset** (95 instances): Problems solved by ≤5 agents - maximum evaluative sensitivity
+   - Combines unsolved, ultra-rare, and very-rare problems
+   - Top agent: 11.6% vs 73.2% on full benchmark (6x better discrimination)
+
+2. **⚡ Challenging Subset** (155 instances): Problems solved by ≤20 agents - strong evaluative power  
+   - Balances discrimination with statistical significance
+   - Includes frontier + rare and uncommon problems
+
+3. **💪 Hard Subset** (45 instances): All Hard difficulty problems regardless of solve rate
+   - Traditional difficulty-based evaluation
+   - Focuses on problems originally classified as most difficult
+
+4. **📁 MultiFile Subset** (40 instances): Multi-file problems solved by ≤10 agents
+   - Targets real-world complexity requiring coordinated edits
+   - Even leading agents achieve only 10% success rate
+
+## 🔬 Methodology
+
+Subsets were created through systematic analysis of solve distribution across 83 evaluated SWE-agents:
+- Problems solved by fewer agents provide better discrimination
+- Analysis covers submissions from October 2023 to May 2025
+- "Solved" means the agent's fix passed the verification test suite
+
+## 📈 Key Insights
+
+- **Enhanced Resolution**: Frontier subset provides 6x better discrimination between top systems
+- **Multi-file Complexity**: Represents genuine software engineering challenges
+- **Statistical Significance**: Challenging subset offers robust evaluation with strong discrimination
+- **Real Progress**: Performance on these subsets indicates genuine capability advances
+
+## 🔗 Resources
+
+- **Blog Post**: [From 73% to 11%: Revealing True SWE-Agent Capabilities](https://jatinganhotra.dev/blog/swe-agents/2025/06/05/swe-bench-verified-discriminative-subsets.html)
+- **Dataset**: [SWE-bench_Verified-discriminative](https://huggingface.co/datasets/jatinganhotra/SWE-bench_Verified-discriminative)
+- **Original SWE-Bench**: [SWE-bench.com](https://www.swebench.com/)
+
+## 🚀 Usage
+
+```python
+from datasets import load_dataset
+
+# Load specific discriminative subset
+frontier = load_dataset("jatinganhotra/SWE-bench_Verified-discriminative", split="frontier")
+challenging = load_dataset("jatinganhotra/SWE-bench_Verified-discriminative", split="challenging")
+```
+
+Created by [Jatin Ganhotra](https://jatinganhotra.dev) | Last Updated: June 19 2025

app.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+"""
+Main entry point for the SWE-Bench Verified Discriminative Subsets Leaderboard
+HuggingFace Spaces Entry Point
+"""
+
+# Import the main application from the organized structure
+from src.core.app import create_leaderboard, start_automated_updates
+
+if __name__ == "__main__":
+    import tempfile
+    from pathlib import Path
+    
+    # Ensure experiments directory exists before starting the app
+    experiments_dir = Path("leaderboard_swe_bench_experiments")
+    if not experiments_dir.exists():
+        print("🔄 Experiments directory not found, cloning repository...")
+        try:
+            from src.utils.automated_updater import AutomatedUpdater
+            updater = AutomatedUpdater()
+            if updater.clone_or_update_experiments_repo():
+                print("✅ Successfully cloned experiments repository")
+            else:
+                print("❌ Failed to clone experiments repository")
+        except Exception as e:
+            print(f"❌ Error setting up experiments repository: {e}")
+    else:
+        print("✅ Experiments directory found")
+    
+    # Start automated updates
+    try:
+        start_automated_updates()
+        print("✅ Automated updates started")
+    except Exception as e:
+        print(f"⚠️ Could not start automated updates: {e}")
+    
+    # Create and launch the leaderboard
+    demo = create_leaderboard()
+    demo.launch(
+        server_name="0.0.0.0",
+        server_port=7860,
+        share=False,
+        allowed_paths=[tempfile.gettempdir()]
+    )

config/requirements.txt

@@ -0,0 +1,7 @@
+gradio==4.44.0
+pandas>=2.1.0
+numpy>=1.24.0
+datasets>=2.14.0
+requests>=2.28.0
+beautifulsoup4>=4.12.0
+schedule>=1.2.0

requirements.txt

@@ -0,0 +1,7 @@
+gradio>=5.34.2
+pandas>=2.1.0
+numpy>=1.24.0
+datasets>=2.14.0
+requests>=2.28.0
+schedule>=1.2.0
+PyYAML>=6.0

src/core/app.py

@@ -0,0 +1,1355 @@
+import gradio as gr
+import pandas as pd
+from src.data.data_processor_dynamic import (
+    load_actual_performance_data, 
+    get_subset_descriptions,
+    get_top_10_data,
+    get_2025_data,
+    get_2024_data,
+    get_pre_2024_data,
+    create_html_table_for_subset
+)
+from src.utils.export_utils import (
+    export_full_leaderboard_csv,
+    export_full_leaderboard_json,
+    export_subset_csv,
+    export_subset_json
+)
+from src.utils.automated_updater import get_update_status, get_last_update_info, should_show_update_notification
+from src.utils.update_scheduler import start_automated_updates, force_update, get_scheduler_status
+
+def get_update_status_message() -> str:
+    """Get update status message for display in UI"""
+    try:
+        last_update = get_last_update_info()
+        update_status = get_update_status()
+        
+        # Format last update time
+        last_update_time = last_update.get("last_successful_update")
+        if last_update_time:
+            from datetime import datetime
+            try:
+                dt = datetime.fromisoformat(last_update_time)
+                formatted_time = dt.strftime("%B %d, %Y at %H:%M UTC")
+            except:
+                formatted_time = last_update_time
+        else:
+            formatted_time = "Never"
+        
+        # Check for new submissions
+        new_count = update_status.get("new_submissions_count", 0)
+        
+        base_message = f"**Last Updated**: {formatted_time}"
+        
+        if new_count > 0:
+            base_message += f"\n\n✅ **{new_count} new submissions** detected and automatically processed."
+        
+        return base_message
+        
+    except Exception as e:
+        return f"**Last Updated**: Status unavailable (error: {e})"
+
+def force_update_action():
+    """Force an immediate update and return status with detailed reporting"""
+    try:
+        from src.utils.automated_updater import AutomatedUpdater
+        
+        result = force_update()
+        
+        # Get detailed information for the update
+        updater = AutomatedUpdater()
+        
+        # Count directories in experiments repo
+        experiments_dir = updater.experiments_dir / "evaluation" / "verified"
+        directory_count = 0
+        metadata_count = 0
+        missing_metadata = []
+        
+        if experiments_dir.exists():
+            for d in experiments_dir.iterdir():
+                if d.is_dir() and (d / "results" / "results.json").exists():
+                    directory_count += 1
+                    metadata_file = d / "metadata.yaml"
+                    if metadata_file.exists():
+                        metadata_count += 1
+                    else:
+                        missing_metadata.append(d.name)
+        
+        # Build detailed message
+        if result["success"]:
+            main_message = f"✅ Update completed successfully. {result['message']}"
+        else:
+            main_message = f"❌ Update failed: {result['message']}"
+        
+        # Build simplified details for metadata approach
+        details = f"""
+**Data Source Status:**
+- **Experiment Directories**: {directory_count} submissions with valid results.json
+- **Metadata Files**: {metadata_count} directories with metadata.yaml containing official names
+- **Leaderboard Display**: Showing {metadata_count} agents with official names from metadata.yaml
+
+**Agent Name Source**: Official names are read directly from each submission's metadata.yaml file (info.name field), providing the most accurate and up-to-date agent information."""
+
+        if missing_metadata:
+            details += f"""
+
+**Directories Missing metadata.yaml** ({len(missing_metadata)} total):
+{chr(10).join([f'- {system}' for system in missing_metadata[:5]])}
+{'- ... and ' + str(len(missing_metadata) - 5) + ' more' if len(missing_metadata) > 5 else ''}"""
+        
+        return main_message, details.strip()
+        
+    except Exception as e:
+        return f"❌ Update error: {e}", ""
+
+def create_multi_table_html():
+    """Create multiple HTML tables for different time periods"""
+    
+    # Load the raw data
+    performance_data = load_actual_performance_data()
+    
+    # Get data for different time periods
+    top_10_data = get_top_10_data(performance_data)
+    since_2025_data = get_2025_data(performance_data)
+    data_2024 = get_2024_data(performance_data)
+    pre_2024_data = get_pre_2024_data(performance_data)
+    
+    # Create the complete HTML with multiple tables
+    html = """
+    <div class="leaderboard-container">
+        <!-- Top 10 Table (Always Visible) -->
+        <div class="table-section">
+            <h3>🏆 Top 10 Performers (All Time)</h3>
+            {}
+        </div>
+        
+        <!-- Since 2025 Table (Always Visible) -->
+        <div class="table-section">
+            <h3>🚀 Since January 1, 2025 ({} submissions)</h3>
+            {}
+        </div>
+        
+        <!-- 2024 Table (Collapsible) -->
+        <div class="table-section">
+            <details>
+                <summary><h3>📅 2024 Submissions ({} submissions)</h3></summary>
+                {}
+            </details>
+        </div>
+        
+        <!-- Pre-2024 Table (Collapsible) -->
+        <div class="table-section">
+            <details>
+                <summary><h3>📜 Pre-2024 Submissions ({} submissions)</h3></summary>
+                {}
+            </details>
+        </div>
+        
+        <!-- Full Table (Collapsible) -->
+        <div class="table-section">
+            <details>
+                <summary><h3>📊 Complete Leaderboard (All {} submissions)</h3></summary>
+                {}
+            </details>
+        </div>
+    </div>
+    """.format(
+        create_html_table_for_subset(top_10_data, "top10"),
+        len(since_2025_data),
+        create_html_table_for_subset(since_2025_data, "since2025"),
+        len(data_2024),
+        create_html_table_for_subset(data_2024, "data2024"),
+        len(pre_2024_data),
+        create_html_table_for_subset(pre_2024_data, "pre2024"),
+        len(performance_data),
+        create_html_table_for_subset(performance_data, "complete")
+    )
+    
+    return html
+
+# Export functions for Gradio - Fixed versions
+def download_full_csv():
+    """Export full leaderboard data as CSV"""
+    try:
+        performance_data = load_actual_performance_data()
+        return export_full_leaderboard_csv(performance_data)
+    except Exception as e:
+        print(f"Error in download_full_csv: {e}")
+        # Return a dummy file on error
+        import tempfile, os
+        temp_path = os.path.join(tempfile.gettempdir(), "error.txt")
+        with open(temp_path, 'w') as f:
+            f.write(f"Error: {e}")
+        return temp_path
+
+def download_full_json():
+    """Export full leaderboard data as JSON"""
+    try:
+        performance_data = load_actual_performance_data()
+        return export_full_leaderboard_json(performance_data)
+    except Exception as e:
+        print(f"Error in download_full_json: {e}")
+        import tempfile, os
+        temp_path = os.path.join(tempfile.gettempdir(), "error.txt")
+        with open(temp_path, 'w') as f:
+            f.write(f"Error: {e}")
+        return temp_path
+
+def download_top10_csv():
+    """Export top 10 agents as CSV"""
+    try:
+        performance_data = load_actual_performance_data()
+        top_10_data = get_top_10_data(performance_data)
+        return export_full_leaderboard_csv(top_10_data)
+    except Exception as e:
+        print(f"Error in download_top10_csv: {e}")
+        import tempfile, os
+        temp_path = os.path.join(tempfile.gettempdir(), "error.txt")
+        with open(temp_path, 'w') as f:
+            f.write(f"Error: {e}")
+        return temp_path
+
+def download_2025_csv():
+    """Export 2025 submissions as CSV"""
+    try:
+        performance_data = load_actual_performance_data()
+        data_2025 = get_2025_data(performance_data)
+        return export_full_leaderboard_csv(data_2025)
+    except Exception as e:
+        print(f"Error in download_2025_csv: {e}")
+        import tempfile, os
+        temp_path = os.path.join(tempfile.gettempdir(), "error.txt")
+        with open(temp_path, 'w') as f:
+            f.write(f"Error: {e}")
+        return temp_path
+
+def download_frontier_csv():
+    """Export frontier subset results as CSV"""
+    try:
+        performance_data = load_actual_performance_data()
+        return export_subset_csv(performance_data, "frontier")
+    except Exception as e:
+        print(f"Error in download_frontier_csv: {e}")
+        import tempfile, os
+        temp_path = os.path.join(tempfile.gettempdir(), "error.txt")
+        with open(temp_path, 'w') as f:
+            f.write(f"Error: {e}")
+        return temp_path
+
+def download_challenging_csv():
+    """Export challenging subset results as CSV"""
+    try:
+        performance_data = load_actual_performance_data()
+        return export_subset_csv(performance_data, "challenging")
+    except Exception as e:
+        print(f"Error in download_challenging_csv: {e}")
+        import tempfile, os
+        temp_path = os.path.join(tempfile.gettempdir(), "error.txt")
+        with open(temp_path, 'w') as f:
+            f.write(f"Error: {e}")
+        return temp_path
+
+def download_hard_csv():
+    """Export hard subset results as CSV"""
+    try:
+        performance_data = load_actual_performance_data()
+        return export_subset_csv(performance_data, "hard")
+    except Exception as e:
+        print(f"Error in download_hard_csv: {e}")
+        import tempfile, os
+        temp_path = os.path.join(tempfile.gettempdir(), "error.txt")
+        with open(temp_path, 'w') as f:
+            f.write(f"Error: {e}")
+        return temp_path
+
+def download_multifile_csv():
+    """Export multifile subset results as CSV"""
+    try:
+        performance_data = load_actual_performance_data()
+        return export_subset_csv(performance_data, "multifile")
+    except Exception as e:
+        print(f"Error in download_multifile_csv: {e}")
+        import tempfile, os
+        temp_path = os.path.join(tempfile.gettempdir(), "error.txt")
+        with open(temp_path, 'w') as f:
+            f.write(f"Error: {e}")
+        return temp_path
+
+def create_leaderboard():
+    """Create the main leaderboard interface with filtering"""
+    
+    # Ensure experiments directory exists before creating the leaderboard
+    from pathlib import Path
+    experiments_dir = Path("leaderboard_swe_bench_experiments")
+    if not experiments_dir.exists():
+        print("🔄 Experiments directory not found, cloning repository...")
+        try:
+            from src.utils.automated_updater import AutomatedUpdater
+            updater = AutomatedUpdater()
+            if updater.clone_or_update_experiments_repo():
+                print("✅ Successfully cloned experiments repository")
+            else:
+                print("❌ Failed to clone experiments repository")
+        except Exception as e:
+            print(f"❌ Error setting up experiments repository: {e}")
+    
+    subset_descriptions = get_subset_descriptions()
+    
+    # Create the Gradio interface
+    with gr.Blocks(
+        title="SWE-Bench Discriminative Subsets Leaderboard",
+        theme=gr.themes.Soft(),
+        css="""
+        .gradio-container {
+            max-width: 1600px !important;
+            padding: 0 10px !important;
+            font-family: 'Inter', 'Segoe UI', 'Roboto', -apple-system, BlinkMacSystemFont, sans-serif;
+        }
+        
+        /* Custom HTML table styling - no Bootstrap */
+        .custom-table-container {
+            max-width: 900px;
+            margin: 0 auto;
+            overflow-x: auto;
+        }
+        
+        .custom-leaderboard-table {
+            width: 100%;
+            border-collapse: collapse;
+            font-family: 'Inter', sans-serif;
+            font-size: 14px;
+            margin: 0;
+            padding: 0;
+        }
+        
+        .custom-leaderboard-table th {
+            font-size: 14px;
+            padding: 10px 4px;
+            line-height: 1.3;
+            color: white;
+            font-weight: 600;
+            text-align: center;
+            cursor: pointer;
+            user-select: none;
+            border: 1px solid #ddd;
+        }
+        
+        .custom-leaderboard-table td {
+            font-size: 14px;
+            padding: 8px 4px;
+            line-height: 1.3;
+            text-align: center;
+            border: 1px solid #ddd;
+        }
+        
+        /* Column-specific cell backgrounds (lighter versions of header colors) */
+        .custom-leaderboard-table td.rank-col,
+        .custom-leaderboard-table td.agent-col,
+        .custom-leaderboard-table td.date-col { 
+            background-color: rgba(109, 103, 107, 0.1);
+        }
+        .custom-leaderboard-table td.full-col { 
+            background-color: rgba(52, 152, 219, 0.1);
+        }
+        .custom-leaderboard-table td.frontier-col { 
+            background-color: rgba(231, 76, 60, 0.1);
+        }
+        .custom-leaderboard-table td.challenging-col { 
+            background-color: rgba(243, 156, 18, 0.1);
+        }
+        .custom-leaderboard-table td.hard-col { 
+            background-color: rgba(155, 89, 182, 0.1);
+        }
+        .custom-leaderboard-table td.multifile-col { 
+            background-color: rgba(26, 188, 156, 0.1);
+        }
+        
+        /* Column-specific widths and styling */
+        .rank-col { width: 60px; }
+        .agent-col { width: 220px; text-align: left !important; padding-left: 8px !important; }
+        .date-col { width: 100px; }
+        .full-col { width: 100px; }
+        .frontier-col { width: 80px; }
+        .challenging-col { width: 100px; }
+        .hard-col { width: 80px; }
+        .multifile-col { width: 90px; }
+        
+        /* Color scheme for headers */
+        .custom-leaderboard-table th.rank-col,
+        .custom-leaderboard-table th.agent-col,
+        .custom-leaderboard-table th.date-col { 
+            background: linear-gradient(135deg, #6d676b, #6d676b); 
+        }
+        .custom-leaderboard-table th.full-col { 
+            background: linear-gradient(135deg, #3498db, #2980b9); 
+        }
+        .custom-leaderboard-table th.frontier-col { 
+            background: linear-gradient(135deg, #e74c3c, #c0392b); 
+        }
+        .custom-leaderboard-table th.challenging-col { 
+            background: linear-gradient(135deg, #f39c12, #e67e22); 
+        }
+        .custom-leaderboard-table th.hard-col { 
+            background: linear-gradient(135deg, #9b59b6, #8e44ad); 
+        }
+        .custom-leaderboard-table th.multifile-col { 
+            background: linear-gradient(135deg, #1abc9c, #16a085); 
+        }
+        
+        /* Table section styling */
+        .table-section {
+            margin: 20px 0;
+        }
+        
+        .table-section h3 {
+            margin-bottom: 10px;
+            margin-top: 5px;
+            color: var(--body-text-color, #333);
+        }
+        
+        /* Collapsible sections */
+        details {
+            margin: 20px 0;
+        }
+        
+        details summary {
+            cursor: pointer;
+            padding: 10px;
+            background-color: rgba(248, 249, 250, 0.8);
+            border-radius: 8px;
+            border: 1px solid #ddd;
+            font-weight: bold;
+        }
+        
+        details summary:hover {
+            background-color: rgba(240, 240, 240, 0.9);
+        }
+        
+        details[open] summary {
+            margin-bottom: 15px;
+        }
+        
+        /* Dark theme for collapsible sections */
+        .dark details summary {
+            background-color: rgba(64, 68, 75, 0.8);
+            border-color: #495057;
+            color: var(--body-text-color, #fff);
+        }
+        
+        .dark details summary:hover {
+            background-color: rgba(80, 85, 95, 0.9);
+        }
+        
+        /* Tweet-style blockquotes */
+        blockquote {
+            border-left: 4px solid #1da1f2;
+            background: linear-gradient(135deg, rgba(29, 161, 242, 0.1), rgba(29, 161, 242, 0.05));
+            margin: 20px 0;
+            padding: 20px;
+            font-style: normal;
+            border-radius: 8px;
+            box-shadow: none;
+            box-sizing: border-box;
+            width: 100%;
+            max-width: 100%;
+        }
+        
+        .dark blockquote {
+            border-left-color: #4dabf7;
+            background: linear-gradient(135deg, rgba(77, 171, 247, 0.15), rgba(77, 171, 247, 0.08));
+        }
+        
+        /* Row styling */
+        .custom-leaderboard-table tbody tr:nth-child(even) {
+            background-color: #fafafa;
+        }
+        
+        .custom-leaderboard-table tbody tr:hover {
+            background-color: #f0f0f0;
+        }
+        
+        /* Dark theme compatibility */
+        .dark .custom-leaderboard-table th {
+            border-color: #495057;
+        }
+        
+        .dark .custom-leaderboard-table td {
+            border-color: #495057;
+            background-color: #2c3e50;
+            color: #ecf0f1;
+        }
+        
+        .dark .custom-leaderboard-table tbody tr:nth-child(even) {
+            background-color: #34495e;
+        }
+        
+        .dark .custom-leaderboard-table tbody tr:hover {
+            background-color: #3c4f66;
+        }
+        
+        
+        /* Light theme styles */
+        .subset-description {
+            background-color: rgba(248, 249, 250, 0.8);
+            color: var(--body-text-color, #333);
+            padding: 20px;
+            border-radius: 8px;
+            margin: 15px 0;
+            border-left: 4px solid #007acc;
+            box-shadow: none;
+            box-sizing: border-box;
+            width: 100%;
+            max-width: 100%;
+        }
+        
+        /* Individual subset styling - Extra compact version */
+        .frontier-subset {
+            background: linear-gradient(135deg, rgba(231, 76, 60, 0.08), rgba(231, 76, 60, 0.04));
+            border-left: 4px solid #e74c3c;
+            border-radius: 6px;
+            padding: 8px 12px;
+            margin: 5px 0;
+            box-shadow: 0 1px 3px rgba(231, 76, 60, 0.1);
+            font-size: 12px;
+            line-height: 1.3;
+        }
+        
+        .challenging-subset {
+            background: linear-gradient(135deg, rgba(243, 156, 18, 0.08), rgba(243, 156, 18, 0.04));
+            border-left: 4px solid #f39c12;
+            border-radius: 6px;
+            padding: 8px 12px;
+            margin: 5px 0;
+            box-shadow: 0 1px 3px rgba(243, 156, 18, 0.1);
+            font-size: 12px;
+            line-height: 1.3;
+        }
+        
+        .hard-subset {
+            background: linear-gradient(135deg, rgba(155, 89, 182, 0.08), rgba(155, 89, 182, 0.04));
+            border-left: 4px solid #9b59b6;
+            border-radius: 6px;
+            padding: 8px 12px;
+            margin: 5px 0;
+            box-shadow: 0 1px 3px rgba(155, 89, 182, 0.1);
+            font-size: 12px;
+            line-height: 1.3;
+        }
+        
+        .multifile-subset {
+            background: linear-gradient(135deg, rgba(26, 188, 156, 0.08), rgba(26, 188, 156, 0.04));
+            border-left: 4px solid #1abc9c;
+            border-radius: 6px;
+            padding: 8px 12px;
+            margin: 5px 0;
+            box-shadow: 0 1px 3px rgba(26, 188, 156, 0.1);
+            font-size: 12px;
+            line-height: 1.3;
+        }
+        
+        .subset-title {
+            font-size: 18px;
+            font-weight: 700;
+            margin-bottom: 8px;
+            display: flex;
+            align-items: center;
+            gap: 8px;
+        }
+        
+        .subset-subtitle {
+            font-size: 14px;
+            font-weight: 600;
+            margin-bottom: 12px;
+            opacity: 0.8;
+        }
+        
+        .subset-details {
+            font-size: 14px;
+            line-height: 1.6;
+            margin-top: 12px;
+        }
+        .methodology-section {
+            background-color: rgba(232, 244, 248, 0.8);
+            color: var(--body-text-color, #333);
+            padding: 20px;
+            border-radius: 8px;
+            margin: 20px 0;
+            border-left: 4px solid #28a745;
+            box-shadow: none;
+            box-sizing: border-box;
+            width: 100%;
+            max-width: 100%;
+        }
+        .insights-section {
+            background-color: rgba(255, 243, 205, 0.8);
+            color: var(--body-text-color, #333);
+            padding: 20px;
+            border-radius: 8px;
+            margin: 20px 0;
+            border-left: 4px solid #ffc107;
+            box-shadow: none;
+            box-sizing: border-box;
+            width: 100%;
+            max-width: 100%;
+        }
+        
+        /* Dark theme compatibility */
+        .dark .subset-description {
+            background-color: rgba(64, 68, 75, 0.8);
+            color: var(--body-text-color, #fff);
+            border-left: 5px solid #4dabf7;
+        }
+        
+        .dark .frontier-subset {
+            background: linear-gradient(135deg, rgba(231, 76, 60, 0.15), rgba(231, 76, 60, 0.08));
+            border-left: 4px solid #ff6b6b;
+            color: var(--body-text-color, #fff);
+        }
+        
+        .dark .challenging-subset {
+            background: linear-gradient(135deg, rgba(243, 156, 18, 0.15), rgba(243, 156, 18, 0.08));
+            border-left: 4px solid #ffd43b;
+            color: var(--body-text-color, #fff);
+        }
+        
+        .dark .hard-subset {
+            background: linear-gradient(135deg, rgba(155, 89, 182, 0.15), rgba(155, 89, 182, 0.08));
+            border-left: 4px solid #b197fc;
+            color: var(--body-text-color, #fff);
+        }
+        
+        .dark .multifile-subset {
+            background: linear-gradient(135deg, rgba(26, 188, 156, 0.15), rgba(26, 188, 156, 0.08));
+            border-left: 4px solid #51cf66;
+            color: var(--body-text-color, #fff);
+        }
+        .dark .methodology-section {
+            background-color: rgba(37, 56, 64, 0.8);
+            color: var(--body-text-color, #fff);
+            border-left: 5px solid #51cf66;
+        }
+        .dark .insights-section {
+            background-color: rgba(77, 63, 33, 0.8);
+            color: var(--body-text-color, #fff);
+            border-left: 5px solid #ffd43b;
+        }
+        
+        .header {
+            text-align: center;
+            padding: 8px 0;
+            font-family: 'Inter', sans-serif;
+        }
+        
+        /* Update Status Styling */
+        .update-status {
+            background-color: rgba(240, 248, 255, 0.8);
+            border-left: 4px solid #007acc;
+            padding: 15px;
+            border-radius: 8px;
+            margin: 10px 0;
+            font-size: 14px;
+        }
+        
+        .dark .update-status {
+            background-color: rgba(37, 56, 64, 0.8);
+            border-left: 4px solid #4dabf7;
+            color: var(--body-text-color, #fff);
+        }
+        
+        .update-message {
+            padding: 15px;
+            border-radius: 8px;
+            margin: 15px 0;
+            font-size: 14px;
+            border-left: 4px solid #28a745;
+            background-color: rgba(40, 167, 69, 0.1);
+        }
+        
+        .update-details {
+            padding: 15px;
+            border-radius: 8px;
+            margin: 15px 0;
+            font-size: 13px;
+            border-left: 4px solid #6c757d;
+            background-color: rgba(108, 117, 125, 0.1);
+            font-family: 'Monaco', 'Consolas', monospace;
+        }
+        
+        .dark .update-message {
+            background-color: rgba(40, 167, 69, 0.15);
+            color: var(--body-text-color, #fff);
+        }
+        
+        .dark .update-details {
+            background-color: rgba(108, 117, 125, 0.15);
+            color: var(--body-text-color, #fff);
+        }
+        
+        /* Enhanced update button styling */
+        .update-button {
+            background: linear-gradient(135deg, #007bff, #0056b3) !important;
+            border: none !important;
+            padding: 12px 24px !important;
+            font-weight: 600 !important;
+            font-size: 16px !important;
+            border-radius: 8px !important;
+            box-shadow: 0 4px 6px rgba(0, 123, 255, 0.25) !important;
+            transition: all 0.3s ease !important;
+        }
+        
+        .update-button:hover {
+            background: linear-gradient(135deg, #0056b3, #004085) !important;
+            transform: translateY(-2px) !important;
+            box-shadow: 0 6px 12px rgba(0, 123, 255, 0.35) !important;
+        }
+        
+        .update-button:active {
+            transform: translateY(0) !important;
+        }
+        .footer {
+            text-align: center;
+            color: var(--body-text-color-subdued, #666);
+            font-size: 14px;
+            margin-top: 20px;
+            font-family: 'Inter', sans-serif;
+        }
+        .tab-content {
+            padding: 0px 0;
+            width: 100%;
+            max-width: 100%;
+            box-sizing: border-box;
+        }
+        .image-container {
+            text-align: center;
+            margin: 20px 0;
+            padding: 15px;
+            border-radius: 8px;
+            background-color: rgba(255, 255, 255, 0.5);
+        }
+        .dark .image-container {
+            background-color: rgba(40, 44, 52, 0.5);
+        }
+        
+        /* Responsive layout for mobile and tablet */
+        @media (max-width: 768px) {
+            .gradio-container {
+                padding: 0 10px !important;
+            }
+            
+            .subset-description,
+            .methodology-section,
+            .insights-section,
+            blockquote {
+                margin: 15px 0;
+                padding: 15px;
+            }
+            
+            .tab-content {
+                padding: 15px 0;
+            }
+        }
+        
+        @media (max-width: 480px) {
+            .gradio-container {
+                padding: 0 5px !important;
+            }
+            
+            .subset-description,
+            .methodology-section,
+            .insights-section,
+            blockquote {
+                margin: 10px 0;
+                padding: 12px;
+            }
+        }
+        </style>
+        """,
+        js="""
+        function() {
+            // Define sortTable function in global scope
+            window.sortTable = function(n, tableId) {
+                var table = document.getElementById(tableId);
+                if (!table) {
+                    console.log('Table not found:', tableId);
+                    return;
+                }
+                
+                var tbody = table.querySelector('tbody');
+                if (!tbody) {
+                    console.log('Tbody not found in table:', tableId);
+                    return;
+                }
+                
+                var rows = Array.from(tbody.rows);
+                var ascending = table.getAttribute('data-sort-col') !== n.toString() || table.getAttribute('data-sort-dir') === 'desc';
+                
+                rows.sort(function(a, b) {
+                    var aVal = a.cells[n].textContent.trim();
+                    var bVal = b.cells[n].textContent.trim();
+                    
+                    // Extract numeric value from percentage strings (e.g., "73.2%" -> 73.2)
+                    var aNum = parseFloat(aVal.replace('%', '').split('(')[0].trim());
+                    var bNum = parseFloat(bVal.replace('%', '').split('(')[0].trim());
+                    
+                    // If both are numbers, compare numerically
+                    if (!isNaN(aNum) && !isNaN(bNum)) {
+                        if (aNum < bNum) return ascending ? -1 : 1;
+                        if (aNum > bNum) return ascending ? 1 : -1;
+                        return 0;
+                    }
+                    
+                    // Otherwise compare as strings
+                    if (aVal < bVal) return ascending ? -1 : 1;
+                    if (aVal > bVal) return ascending ? 1 : -1;
+                    return 0;
+                });
+                
+                // Re-append sorted rows
+                rows.forEach(function(row) {
+                    tbody.appendChild(row);
+                });
+                
+                // Update sort indicators
+                table.setAttribute('data-sort-col', n);
+                table.setAttribute('data-sort-dir', ascending ? 'asc' : 'desc');
+                
+                // Visual feedback
+                var headers = table.querySelectorAll('th');
+                headers.forEach(function(header, index) {
+                    header.style.opacity = index === n ? '1.0' : '0.7';
+                });
+            };
+        }
+        """
+    ) as demo:
+        
+        # Header - More compact
+        gr.Markdown(
+            """
+            # 🏆 SWE-Bench Verified Discriminative Subsets Leaderboard
+            
+            **Top SWE-agents claim high success rates, but drop dramatically on truly challenging problems**
+            
+            📖 **Full Analysis**: [From 73% to 11%: Revealing True SWE-Agent Capabilities](https://jatinganhotra.dev/blog/swe-agents/2025/06/05/swe-bench-verified-discriminative-subsets.html) | 📊 **Dataset**: [SWE-bench_Verified-discriminative](https://huggingface.co/datasets/jatinganhotra/SWE-bench_Verified-discriminative)
+            """,
+            elem_classes=["header"]
+        )
+        
+        
+        # Create tabs
+        with gr.Tabs():
+            # Tab 1: Performance Rankings
+            with gr.TabItem("📈 Performance Rankings"):
+                with gr.Column(elem_classes=["tab-content"]):
+                    gr.Markdown(
+                        """
+                        🚨 SWE-Bench Verified is saturated. When every agent solves 160+ of the same "easy" problems, how do we know which is actually better? Our discriminative subsets target the competitive frontier where top systems actually differ and their performance drops dramatically. Explore the methodology in the **"📊 Discriminative Subsets Explained"** tab.
+                        
+                        **NOTE**: _These aren't just harder problems—they reveal what SWE agents actually can't do today._ 
+
+                        **Subset Definitions**:
+                        - 🔥 **Frontier** (95 instances): Problems solved by ≤5 agents (maximum discrimination)
+                        - ⚡ **Challenging** (155 instances): Problems solved by ≤20 agents (balanced evaluation)  
+                        - 💪 **Hard** (45 instances): All Hard difficulty problems (traditional difficulty)
+                        - 📁 **MultiFile** (40 instances): Multi-file problems solved by ≤10 agents (real-world complexity)
+                        
+                        How to Read: The results are ranked by full benchmark performance. Click on the column headers to sort by subset performance.
+                        """
+                    )
+                    
+                    # Multi-table HTML with time-based filtering
+                    table_html = create_multi_table_html()
+                    gr.HTML(table_html)
+            
+            # Tab 2: Discriminative Subsets Explained
+            with gr.TabItem("📊 Discriminative Subsets Explained"):
+                with gr.Column(elem_classes=["tab-content"]):
+                    gr.Markdown(
+                        """
+                        ## The Emperor's New Benchmarks
+                        
+                        Like the tale of the Emperor's new clothes, sometimes we need fresh eyes on familiar benchmarks. 
+                        SWE-agents parade around with impressive scores, but here's the reality:
+                        
+                        - **Everyone's solving the same problems**: 69% of SWE-Bench Verified is solved by 21+ agents
+                        - **Making most evaluations meaningless**: When everyone gets the same questions right, you can't tell who's actually better  
+                        - **The naked truth**: Many problems remain completely unsolved by ALL agents
+                        
+                        **What everyone sees**: "Our agent achieves impressive results on SWE-Bench!"  
+                        **The reality**: It's the same set of 350 problems everyone solves.
+                        
+                        📖 [Full breakdown of the saturation problem](https://jatinganhotra.dev/blog/swe-agents/2025/06/05/swe-bench-verified-discriminative-subsets.html)
+                        """
+                    )
+                    
+                    gr.Markdown("> 💡 **Key Insight**: Instead of arbitrary 'hard' problems, analyze HOW MANY agents solve each instance across 500 problems.")
+                    
+                    gr.Markdown(
+                        """
+                        ## A Data-Driven Solution: Discriminative Subsets
+                        
+                        **Instead of guessing which problems are "hard", I analyzed how many SWE-agents solve each instance.**
+                        
+                        Rather than arbitrarily choosing "hard" problems, I developed a systematic methodology by analyzing 
+                        how many SWE-agents solve each instance across all 500 problems in SWE-Bench Verified.
+                        
+                        ### Instance Solve Distribution Analysis
+                        
+                        Each of the 500 instances was checked against evaluation results from all SWE-agents 
+                        (submitted to the leaderboard) to record solve counts. "Solved" means the agent's fix 
+                        passed the verification test suite - the same standard used in the original SWE-bench evaluation.
+                        
+                        📊 [Explore the discriminative dataset](https://huggingface.co/datasets/jatinganhotra/SWE-bench_Verified-discriminative)
+                        """
+                    )
+                    
+                    # Image 1: Instance Solve Count Distribution
+                    with gr.Column(elem_classes=["image-container"]):
+                        gr.Image(
+                            "assets/Instance Solve Count Distribution.png",
+                            label="Instance Solve Count Distribution",
+                            show_label=True,
+                            show_download_button=False,
+                            height=400
+                        )
+                        gr.Markdown("> 📊 **Key Finding**: 69% of problems are solved by 21+ agents, creating limited evaluative sensitivity.")
+                    
+                    gr.Markdown(
+                        """
+                        I categorized all instances based on how many of the 83 evaluated SWE-agents successfully solve them:
+
+                        | Bucket | Solve Count | Instances | Percentage | Easy | Medium | Hard | Single | Multi |
+                        |--------|-------------|-----------|------------|------|--------|------|--------|-------|
+                        | **Unsolved** | 0 agents | 52 | 10.4% | 5 | 26 | 21 | 27 | 25 |
+                        | **Ultra Rare** | 1-2 agents | 26 | 5.2% | 6 | 16 | 4 | 17 | 9 |
+                        | **Very Rare** | 3-5 agents | 17 | 3.4% | 3 | 10 | 4 | 14 | 3 |
+                        | **Rare** | 6-10 agents | 22 | 4.4% | 1 | 19 | 2 | 19 | 3 |
+                        | **Uncommon** | 11-20 agents | 38 | 7.6% | 13 | 22 | 3 | 28 | 10 |
+                        | **Common** | 21-40 agents | 96 | 19.2% | 27 | 62 | 7 | 82 | 14 |
+                        | **Very Common** | 41-60 agents | 93 | 18.6% | 38 | 52 | 3 | 88 | 5 |
+                        | **Solved** | 61+ agents | 156 | 31.2% | 101 | 53 | 2 | 154 | 2 |
+                        
+                        > 🎯 **The Solution**: Four discriminative subsets targeting the competitive frontier where agents actually differ.
+                        """
+                    )
+                    
+                    # Four Discriminative Subsets section
+                    gr.Markdown(
+                        """
+                        ### Four Discriminative Subsets: Targeting the Competitive Frontier
+                        
+                        **The Solution: 4 targeted subsets that reveal true agent capabilities.**
+                        
+                        Rather than continuing to measure incremental improvements on largely-solved Easy problems, 
+                        I designed targeted subsets to focus on:
+                        
+                        1. **Completely unsolved problems** (52 instances) - true frontier challenges that NO agent can solve
+                        2. **Sparsely solved problems** - instances resolved by only a handful of agents  
+                        3. **Problems with high solution variance** - where top SWE-agents show meaningful differences
+                        4. **Real-world complexity** - multi-file coordination that reflects actual software engineering
+                        
+                        📖 [Deep dive into each subset's methodology](https://jatinganhotra.dev/blog/swe-agents/2025/06/05/swe-bench-verified-discriminative-subsets.html)
+                        """
+                    )
+                    
+                    # Image 2: Performance Comparison
+                    with gr.Column(elem_classes=["image-container"]):
+                        gr.Image(
+                            "assets/Performance Comparison Across Subsets.png",
+                            label="Performance Comparison Across Subsets",
+                            show_label=True,
+                            show_download_button=False,
+                            height=400
+                        )
+                        gr.Markdown("> 🔥 **Breakthrough Result**: Top agents drop dramatically on Frontier subset - significantly better discrimination!")
+                    
+                    # Four Discriminative Subsets - Compact Grid Layout
+                    gr.Markdown("### 🎯 The Four Discriminative Subsets")
+                    
+                    with gr.Row():
+                        with gr.Column(scale=1):
+                            gr.Markdown(
+                                """
+                                **🔥 Frontier Subset** (95 instances)  
+                                Problems solved by ≤5 agents - maximum discrimination  
+                                *Combines unsolved, ultra-rare, and very-rare problems. Provides 6x better discrimination than full benchmark.*
+                                """,
+                                elem_classes=["frontier-subset"]
+                            )
+                        
+                        with gr.Column(scale=1):
+                            gr.Markdown(
+                                """
+                                **⚡ Challenging Subset** (155 instances)  
+                                Problems solved by ≤20 agents - strong evaluative power  
+                                *Expands Frontier to include rare and uncommon problems. Balances discrimination with statistical significance.*
+                                """,
+                                elem_classes=["challenging-subset"]
+                            )
+                    
+                    with gr.Row():
+                        with gr.Column(scale=1):
+                            gr.Markdown(
+                                """
+                                **💪 Hard Subset** (45 instances)  
+                                All Hard difficulty problems regardless of solve rate  
+                                *Traditional difficulty-based subset. Includes all problems classified as "Hard" in the original SWE-Bench difficulty analysis.*
+                                """,
+                                elem_classes=["hard-subset"]
+                            )
+                        
+                        with gr.Column(scale=1):
+                            gr.Markdown(
+                                """
+                                **📁 MultiFile Subset** (40 instances)  
+                                Multi-file problems solved by ≤10 agents  
+                                *Targets problems requiring coordinated edits across multiple source files - representing real-world software engineering complexity.*
+                                """,
+                                elem_classes=["multifile-subset"]
+                            )
+                    
+                    # Subset summary table
+                    gr.Markdown(
+                        """
+                        ### Subset Summary
+                        
+                        | Subset | Description | Total | Easy | Medium | Hard | Single | Multi | Solve Range | Top Agent % |
+                        |--------|-------------|-------|------|--------|------|--------|-------|-------------|-------------|
+                        | **Frontier** | Solved by ≤5 agents | 95 | 14 | 52 | 29 | 58 | 37 | 0–5 | 11.6% |
+                        | **Challenging** | Solved by ≤20 agents | 155 | 28 | 93 | 34 | 105 | 50 | 0–20 | 31.6% |
+                        | **Hard** | All Hard problems | 45 | 0 | 0 | 45 | 20 | 25 | 0–61 | 42.2% |
+                        | **MultiFile** | Multi-file + ≤10 solves | 40 | 3 | 17 | 20 | 0 | 40 | 0–7 | 10.0% |
+                        """
+                    )
+            
+            # Tab 3: Key Insights
+            with gr.TabItem("🔍 Key Insights"):
+                with gr.Column(elem_classes=["tab-content"]):
+                    gr.Markdown(
+                        """
+                        ## The Results Are Eye-Opening
+                        
+                        **Top agent on full benchmark: Strong performance ✅**  
+                        **Same agent on Frontier subset: Dramatic drop 😬**
+                        
+                        The discriminative subsets expose critical insights about current SWE-agent capabilities that traditional benchmarks completely miss.
+                        
+                        📖 [See the complete performance breakdown](https://jatinganhotra.dev/blog/swe-agents/2025/06/05/swe-bench-verified-discriminative-subsets.html)
+                        """
+                    )
+                    
+                    gr.Markdown("> 📈 **Breakthrough Result**: 6x better discrimination power. Finally, a way to meaningfully compare state-of-the-art SWE agents.")
+                    
+                    gr.Markdown(
+                        """
+                        ### 🎯 Enhanced Resolution Power: From Saturated to Sensitive
+                        - **6x Better Discrimination**: Frontier subset provides much clearer distinction between top systems
+                        - **Dramatic Performance Drops**: Top agents show significant performance decreases on challenging problems  
+                        - **52 Completely Unsolved Problems**: Not edge cases—problems that separate toys from tools
+                        - **Statistical Significance**: Challenging subset balances discrimination with robust evaluation
+                        """
+                    )
+                    
+                    gr.Markdown("> 📁 **Multi-file Challenge**: Multi-file coordination remains the hardest challenge - even top agents struggle at 10% success.")
+                    
+                    gr.Markdown(
+                        """
+                        ### 🔧 Multi-file Problems = Everyday Dev Work, But Agents Flop Here
+                        - **Real-world Challenge**: Even leading agents achieve only 10% on multi-file problems
+                        - **Coordination Required**: Problems need edits across multiple source files simultaneously  
+                        - **Industry Relevance**: Reflects actual software engineering complexity
+                        - **The Gap**: We need agents that can coordinate real code changes, not just single-file tweaks
+                        
+                        ### 🧠 "Hard" Doesn't Mean Hard for AI
+                        
+                        **Key insight**: Traditional difficulty labels miss what makes problems truly challenging for AI agents:
+                        
+                        - **2/45 "Hard" problems** are solved by 61+ agents (too easy for evaluation)
+                        - **5/194 "Easy" problems** are solved by ZERO agents (actually impossible) 
+                        - **52 Medium problems** in Frontier subset vs 29 Hard problems
+                        
+                        We need better ways to measure difficulty—and these subsets provide exactly that.
+                        
+                        ### 📊 Subset Comparison: The New Reality
+                        
+                        | Subset | Instances | Top Performance | Focus | Insight |
+                        |--------|-----------|----------------|-------|---------|
+                        | **Full Benchmark** | 500 | 73.2% | Complete evaluation | Saturated for top agents |
+                        | **Frontier** | 95 | 11.6% | Maximum sensitivity | True cutting-edge capability |
+                        | **Challenging** | 155 | 31.6% | Balanced evaluation | Practical discrimination |
+                        | **Hard** | 45 | 42.2% | Traditional difficulty | Many are actually easy |
+                        | **MultiFile** | 40 | 10.0% | Real-world complexity | Critical capability gap |
+                        """
+                    )
+                    
+                    gr.Markdown("> 🚀 **Looking Ahead**: We need evaluation frameworks that grow with AI capabilities, not static benchmarks.")
+                    
+                    gr.Markdown(
+                        """
+                        ### 🧠 Research Implications: A New Methodology for Benchmark Evolution
+                        
+                        **New methodology for benchmark evolution as AI saturates existing evaluations:**
+                        
+                        1. **Analyze solve distribution** across all systems (not just top performers)
+                        2. **Identify low-solve instances** where systems actually differ 
+                        3. **Create discriminative subsets** targeting competitive frontiers
+                        4. **Maintain evaluative sensitivity** as capabilities advance
+                        
+                        **Applied to SWE-Bench**: Performance drops dramatically on discriminative subsets, exposing real capability gaps.
+                        
+                        - **Progress Tracking**: Improvements on discriminative subsets indicate genuine capability advances
+                        - **Research Focus**: Identifies specific bottlenecks for targeted development  
+                        - **Benchmark Evolution**: Framework can adapt as agent capabilities improve
+                        - **Actionable Insight**: Focus research on sparsely-solved problems, not incrementally improving on saturated ones
+                        """
+                    )
+                    
+                    gr.Markdown("> 🎯 **Challenge for SWE-agent developers**: Can your agent solve ANY of the 52 completely unsolved problems? These aren't edge cases—they're what separate toys from tools.")
+                    
+                    gr.Markdown(
+                        """
+                        ## 🚀 Ready to Benchmark YOUR Agent Properly?
+                        
+                        **Stop optimizing for saturated benchmarks. Start measuring real progress.**
+                        
+                        - **📊 Dataset Available Now**: [SWE-bench_Verified-discriminative](https://huggingface.co/datasets/jatinganhotra/SWE-bench_Verified-discriminative)
+                        - **📖 Full Analysis**: [From 73% to 11%: Revealing True SWE-Agent Capabilities](https://jatinganhotra.dev/blog/swe-agents/2025/06/05/swe-bench-verified-discriminative-subsets.html)
+                        - **🌐 Original SWE-Bench**: [SWE-bench.com](https://www.swebench.com/)
+                        
+                        **The 52 unsolved problems are still out there. Who's going to be first?** 👀
+                        """
+                    )
+            
+            # Tab 4: Auto-Update (moved to last position)
+            with gr.TabItem("🔄 Auto-Update"):
+                with gr.Column(elem_classes=["tab-content"]):
+                    gr.Markdown(
+                        """
+                        # 🔄 Automated Update System
+                        
+                        This tab manages the synchronization with the SWE-Bench experiments repository. 
+                        The system automatically checks for new submissions daily and can be manually triggered below.
+                        
+                        ## How It Works
+                        
+                        1. **Daily Automatic Updates**: The system checks for new submissions every day at 6 AM UTC
+                        2. **Git Repository Sync**: Downloads latest data from the SWE-bench experiments repository
+                        3. **New Submission Detection**: Scans for new directories with valid results.json files and reads agent details directly from each submission's metadata.yaml file
+                        4. **Automatic Integration**: New submissions are immediately available with official names
+                        
+                        ## Update Status
+                        """
+                    )
+                    
+                    # Update Status Display
+                    update_status_display = gr.Markdown(
+                        get_update_status_message(),
+                        elem_classes=["update-status"]
+                    )
+                    
+                    # Update Controls
+                    with gr.Row():
+                        with gr.Column(scale=2):
+                            gr.Markdown("### Manual Update")
+                            gr.Markdown("Click the button below to force an immediate update check and synchronization.")
+                        with gr.Column(scale=1):
+                            force_update_btn = gr.Button(
+                                "🚀 Sync with SWE-Bench Now", 
+                                size="lg",
+                                variant="primary",
+                                elem_classes=["update-button"]
+                            )
+                    
+                    # Update Results Area
+                    update_message = gr.Markdown(
+                        "",
+                        visible=False,
+                        elem_classes=["update-message"]
+                    )
+                    
+                    # Update Details (for discrepancy reporting)
+                    update_details = gr.Markdown(
+                        "",
+                        visible=False,
+                        elem_classes=["update-details"]
+                    )
+                    
+                    gr.Markdown(
+                        """
+                        ## Technical Details
+                        
+                        **Data Sources**:
+                        - **Experiments Repository**: [github.com/SWE-bench/experiments](https://github.com/SWE-bench/experiments)
+                        - **Agent Metadata**: Each submission's `metadata.yaml` file contains official agent names
+                        - **Performance Data**: Results from `results/results.json` in each submission directory
+                    
+                        **Automatic Processing**: all submissions with valid 
+                        `metadata.yaml` and `results/results.json` files are automatically integrated with their official names.
+                        """
+                    )
+        
+            # Tab 5: Downloads
+            with gr.TabItem("📥 Downloads"):
+                with gr.Column(elem_classes=["tab-content"]):
+                    gr.Markdown(
+                        """
+                        # 📥 Download Leaderboard Data
+                        
+                        Export the complete SWE-Bench discriminative subsets leaderboard data in various formats. 
+                        All downloads include comprehensive agent metadata and performance metrics across all subsets.
+                        
+                        **📌 Important**: Double-click the download buttons below to initiate the download. Single clicks may not trigger the download in some browsers.
+                        
+                        **File Formats**:
+                        - **CSV**: Spreadsheet-compatible format for analysis and visualization
+                        - **JSON**: Structured data with metadata for programmatic use
+                        
+                        **Data Included**: Agent names, submission dates, performance across all 5 benchmarks, 
+                        solve counts, and percentage scores.
+                        """
+                    )
+                    
+                    # Complete Leaderboard Section
+                    gr.Markdown("## 📊 Complete Leaderboard")
+                    gr.Markdown("Download the full dataset with all 89+ agents and performance across all subsets.")
+                    
+                    with gr.Row():
+                        full_csv_btn = gr.DownloadButton(
+                            "📄 Download Complete CSV", 
+                            size="lg",
+                            variant="primary"
+                        )
+                        full_json_btn = gr.DownloadButton(
+                            "📋 Download Complete JSON", 
+                            size="lg",
+                            variant="secondary"
+                        )
+                    
+                    # Filtered Exports Section
+                    gr.Markdown("## 🎯 Filtered Exports")
+                    gr.Markdown("Download specific agent groups or time periods.")
+                    
+                    with gr.Row():
+                        with gr.Column():
+                            gr.Markdown("### 🏆 Top Performers")
+                            top10_csv_btn = gr.DownloadButton(
+                                "Top 10 Agents CSV", 
+                                size="sm",
+                                variant="secondary"
+                            )
+                        
+                        with gr.Column():
+                            gr.Markdown("### 🚀 Recent (2025)")
+                            recent_csv_btn = gr.DownloadButton(
+                                "2025 Submissions CSV", 
+                                size="sm",
+                                variant="secondary"
+                            )
+                    
+                    # Individual Subsets Section
+                    gr.Markdown("## 📊 Individual Subset Results")
+                    gr.Markdown("Download performance data for specific discriminative subsets only.")
+                    
+                    with gr.Row():
+                        frontier_csv_btn = gr.DownloadButton(
+                            "🔥 Frontier Subset CSV", 
+                            size="sm",
+                            variant="primary"
+                        )
+                        challenging_csv_btn = gr.DownloadButton(
+                            "⚡ Challenging Subset CSV", 
+                            size="sm",
+                            variant="primary"
+                        )
+                    
+                    with gr.Row():
+                        hard_csv_btn = gr.DownloadButton(
+                            "💪 Hard Subset CSV", 
+                            size="sm",
+                            variant="primary"
+                        )
+                        multifile_csv_btn = gr.DownloadButton(
+                            "📁 MultiFile Subset CSV", 
+                            size="sm",
+                            variant="primary"
+                        )
+                    
+                    # Usage Examples
+                    gr.Markdown(
+                        """
+                        ## 💡 Usage Examples
+                        
+                        **Research & Analysis**:
+                        - Load CSV files into Excel, Google Sheets, or Pandas for statistical analysis
+                        - Compare agent performance trends over time
+                        - Identify patterns in discriminative subset performance
+                        
+                        **Integration**:
+                        - Use JSON exports for programmatic access to leaderboard data
+                        - Integrate with evaluation pipelines and research tools
+                        - Build custom visualizations and comparisons
+                        
+                        **Citation**: If you use this data in research, please cite:
+                        > Ganhotra, J. (2025). "From 73% to 11%: Revealing True SWE-Agent Capabilities". 
+                        > SWE-Bench Verified Discriminative Subsets Analysis.
+                        """
+                    )
+                    
+                    # Connect download buttons to functions with proper DownloadButton behavior
+                    full_csv_btn.click(fn=download_full_csv, outputs=full_csv_btn)
+                    full_json_btn.click(fn=download_full_json, outputs=full_json_btn)
+                    top10_csv_btn.click(fn=download_top10_csv, outputs=top10_csv_btn)
+                    recent_csv_btn.click(fn=download_2025_csv, outputs=recent_csv_btn)
+                    frontier_csv_btn.click(fn=download_frontier_csv, outputs=frontier_csv_btn)
+                    challenging_csv_btn.click(fn=download_challenging_csv, outputs=challenging_csv_btn)
+                    hard_csv_btn.click(fn=download_hard_csv, outputs=hard_csv_btn)
+                    multifile_csv_btn.click(fn=download_multifile_csv, outputs=multifile_csv_btn)
+            
+        # Connect force update button
+        def handle_force_update():
+            message, details = force_update_action()
+            new_status = get_update_status_message()
+            return (
+                new_status, 
+                message, 
+                gr.update(visible=True), 
+                details, 
+                gr.update(visible=True if details else False)
+            )
+        
+        force_update_btn.click(
+            fn=handle_force_update,
+            outputs=[update_status_display, update_message, update_message, update_details, update_details]
+        )
+        
+        # Footer
+        gr.Markdown(
+            """
+            ---
+            **Data Source**: [From 73% to 11%: Revealing True SWE-Agent Capabilities](https://jatinganhotra.dev/blog/swe-agents/2025/06/05/swe-bench-verified-discriminative-subsets.html) • Updated daily to match the SWE-Bench leaderboard  
+            **Last Updated**: June 19, 2025 at 12:00 UTC  
+            **Created by**: [Jatin Ganhotra](https://jatinganhotra.dev)
+            """,
+            elem_classes=["footer"]
+        )
+    
+    return demo
+
+# Create and launch the app
+if __name__ == "__main__":
+    import tempfile
+    
+    # Start automated updates
+    try:
+        start_automated_updates()
+        print("✅ Automated updates started")
+    except Exception as e:
+        print(f"⚠️ Could not start automated updates: {e}")
+    
+    demo = create_leaderboard()
+    demo.launch(
+        server_name="0.0.0.0",
+        server_port=7860,
+        share=False,
+        allowed_paths=[tempfile.gettempdir()]
+    )

src/data/data_processor_dynamic.py

@@ -0,0 +1,216 @@
+import pandas as pd
+import json
+from pathlib import Path
+from src.data.leaderboard_data_fetcher import get_leaderboard_data
+
+def load_actual_performance_data():
+    """
+    Load actual performance data from dynamic leaderboard fetcher
+    """
+    # Get dynamic data from leaderboard fetcher
+    models_data, models_mapping = get_leaderboard_data()
+    
+    # Convert to the format expected by the app
+    performance_data = []
+    
+    for model in models_data:
+        performance_row = {
+            'agent_name': model['agent_name'],
+            'submission_id': model['submission_id'],
+            'submission_date': model.get('submission_date', 'not yet published on leaderboard'),
+            'rank': model['rank'],
+            'full_solved': model['full_solved'],
+            'full_total': model['full_total'],
+            'full_percentage': model['full_percentage'],
+        }
+        
+        # Add subset performance
+        for subset in ['frontier', 'challenging', 'hard', 'multifile']:
+            performance_row[f'{subset}_solved'] = model.get(f'{subset}_solved', 0)
+            performance_row[f'{subset}_total'] = model.get(f'{subset}_total', 0)
+            performance_row[f'{subset}_percentage'] = model.get(f'{subset}_percentage', 0.0)
+        
+        performance_data.append(performance_row)
+    
+    return performance_data
+
+def filter_by_date_range(performance_data, start_date=None, end_date=None):
+    """Filter performance data by date range"""
+    from datetime import datetime
+    
+    filtered_data = []
+    
+    for row in performance_data:
+        submission_date_str = row.get('submission_date', 'not yet published on leaderboard')
+        
+        if submission_date_str == 'not yet published on leaderboard':
+            continue
+            
+        try:
+            submission_date = datetime.strptime(submission_date_str, '%Y-%m-%d')
+            
+            # Check date range
+            if start_date and submission_date < start_date:
+                continue
+            if end_date and submission_date > end_date:
+                continue
+                
+            filtered_data.append(row)
+            
+        except ValueError:
+            # Skip entries with invalid dates
+            continue
+    
+    return filtered_data
+
+def get_top_10_data(performance_data):
+    """Get top 10 performers regardless of date"""
+    return sorted(performance_data, key=lambda x: x['full_percentage'], reverse=True)[:10]
+
+def get_2025_data(performance_data):
+    """Get submissions from Jan 1, 2025 onwards"""
+    from datetime import datetime
+    start_date = datetime(2025, 1, 1)
+    return filter_by_date_range(performance_data, start_date=start_date)
+
+def get_2024_data(performance_data):
+    """Get submissions from 2024 (Jan 1 - Dec 31)"""
+    from datetime import datetime
+    start_date = datetime(2024, 1, 1)
+    end_date = datetime(2024, 12, 31)
+    return filter_by_date_range(performance_data, start_date=start_date, end_date=end_date)
+
+def get_pre_2024_data(performance_data):
+    """Get submissions before 2024"""
+    from datetime import datetime
+    end_date = datetime(2023, 12, 31)
+    return filter_by_date_range(performance_data, end_date=end_date)
+
+def format_for_gradio_table(performance_data):
+    """
+    Format the performance data for display in Gradio table
+    """
+    table_data = []
+    
+    for row in performance_data:
+        formatted_row = [
+            row['rank'],
+            row['agent_name'],
+            row.get('submission_date', 'not yet published on leaderboard'),
+            f"{row['full_percentage']}% ({row['full_solved']}/500)",
+            f"{row['frontier_percentage']}% ({row['frontier_solved']}/{row['frontier_total']})",
+            f"{row['challenging_percentage']}% ({row['challenging_solved']}/{row['challenging_total']})", 
+            f"{row['hard_percentage']}% ({row['hard_solved']}/{row['hard_total']})",
+            f"{row['multifile_percentage']}% ({row['multifile_solved']}/{row['multifile_total']})"
+        ]
+        table_data.append(formatted_row)
+    
+    return table_data
+
+def create_html_table_for_subset(performance_data, table_id, show_rank_column=True):
+    """Create HTML table for a specific subset of data"""
+    
+    if not performance_data:
+        return "<p>No data available for this time period.</p>"
+    
+    # Re-rank the data for this subset
+    sorted_data = sorted(performance_data, key=lambda x: x['full_percentage'], reverse=True)
+    
+    # Create table HTML
+    html = f"""
+    <div class="custom-table-container">
+        <table class="custom-leaderboard-table" id="{table_id}">
+            <thead>
+                <tr>
+    """
+    
+    if show_rank_column:
+        html += '<th onclick="sortTable(0, \'{}\')\" class="rank-col">Rank</th>'.format(table_id)
+        col_offset = 1
+    else:
+        col_offset = 0
+    
+    html += f"""
+                    <th onclick="sortTable({col_offset}, '{table_id}')" class="agent-col">SWE-Agent</th>
+                    <th onclick="sortTable({col_offset + 1}, '{table_id}')" class="date-col">Date</th>
+                    <th onclick="sortTable({col_offset + 2}, '{table_id}')" class="full-col">Full<br>Benchmark</th>
+                    <th onclick="sortTable({col_offset + 3}, '{table_id}')" class="frontier-col">Frontier</th>
+                    <th onclick="sortTable({col_offset + 4}, '{table_id}')" class="challenging-col">Challenging</th>
+                    <th onclick="sortTable({col_offset + 5}, '{table_id}')" class="hard-col">Hard</th>
+                    <th onclick="sortTable({col_offset + 6}, '{table_id}')" class="multifile-col">MultiFile</th>
+                </tr>
+            </thead>
+            <tbody>
+    """
+    
+    # Add data rows
+    for i, row in enumerate(sorted_data):
+        html += '<tr>'
+        
+        if show_rank_column:
+            html += f'<td class="rank-col">{i + 1}</td>'
+        
+        html += f"""
+                    <td class="agent-col">{row['agent_name']}</td>
+                    <td class="date-col">{row.get('submission_date', 'not yet published on leaderboard')}</td>
+                    <td class="full-col">{row['full_percentage']}</td>
+                    <td class="frontier-col">{row['frontier_percentage']}</td>
+                    <td class="challenging-col">{row['challenging_percentage']}</td>
+                    <td class="hard-col">{row['hard_percentage']}</td>
+                    <td class="multifile-col">{row['multifile_percentage']}</td>
+                </tr>
+        """
+    
+    html += """
+            </tbody>
+        </table>
+    </div>
+    """
+    
+    return html
+
+def get_subset_descriptions():
+    """
+    Return descriptions for each subset with detailed methodology
+    """
+    return {
+        'full': {
+            'name': 'SWE-Bench Verified',
+            'description': 'Complete benchmark with all 500 instances from SWE-Bench Verified',
+            'instances': 500,
+            'details': 'The gold standard software engineering benchmark covering real-world GitHub issues'
+        },
+        'frontier': {
+            'name': 'Frontier Subset',
+            'description': 'Problems solved by ≤5 agents - maximum evaluative sensitivity',
+            'instances': 95,
+            'details': 'Combines unsolved (0 agents), ultra-rare (1-2 agents), and very-rare (3-5 agents) problems. Provides 6x better discrimination than full benchmark.'
+        },
+        'challenging': {
+            'name': 'Challenging Subset', 
+            'description': 'Problems solved by ≤20 agents - strong evaluative power',
+            'instances': 155,
+            'details': 'Expands Frontier to include rare (6-10 agents) and uncommon (11-20 agents) problems. Balances discrimination with statistical significance.'
+        },
+        'hard': {
+            'name': 'Hard Subset',
+            'description': 'All Hard difficulty problems regardless of solve rate',
+            'instances': 45,
+            'details': 'Traditional difficulty-based subset. Includes all problems classified as "Hard" in the original SWE-Bench difficulty analysis.'
+        },
+        'multifile': {
+            'name': 'MultiFile Subset',
+            'description': 'Multi-file problems solved by ≤10 agents',
+            'instances': 40,
+            'details': 'Targets problems requiring coordinated edits across multiple source files - representing real-world software engineering complexity.'
+        }
+    }
+
+if __name__ == "__main__":
+    # Test the dynamic data loading
+    data = load_actual_performance_data()
+    print(f"Loaded dynamic data for {len(data)} agents")
+    print("Top 5 agents:")
+    for i in range(min(5, len(data))):
+        agent = data[i]
+        print(f"{i+1}. {agent['agent_name']}: {agent['full_percentage']}% (full), {agent['frontier_percentage']}% (frontier)")

src/data/leaderboard_data_fetcher.py

@@ -0,0 +1,274 @@
+import os
+import json
+import time
+import logging
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+import requests
+from datasets import load_dataset
+import pandas as pd
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+class LeaderboardDataFetcher:
+    """Fetches and processes SWE-bench leaderboard data dynamically"""
+    
+    def __init__(self, cache_dir: str = "leaderboard_cache"):
+        self.cache_dir = Path(cache_dir)
+        self.cache_dir.mkdir(exist_ok=True)
+        self.experiments_dir = Path("leaderboard_swe_bench_experiments/evaluation/verified")
+        
+        # Cache file paths
+        self.models_cache = self.cache_dir / "verified_models.json"
+        self.mapping_cache = self.cache_dir / "verified_models_mapping.json"
+        self.subsets_cache = self.cache_dir / "discriminative_subsets.json"
+        
+    def fetch_with_retry(self, func, max_retries: int = 5, base_delay: float = 1.0):
+        """Execute function with exponential backoff retry logic"""
+        for attempt in range(max_retries):
+            try:
+                return func()
+            except Exception as e:
+                if attempt == max_retries - 1:
+                    logger.error(f"Failed after {max_retries} attempts: {e}")
+                    raise
+                
+                delay = base_delay * (2 ** attempt)
+                logger.warning(f"Attempt {attempt + 1} failed: {e}. Retrying in {delay:.1f}s...")
+                time.sleep(delay)
+    
+    def fetch_discriminative_subsets(self) -> Dict:
+        """Fetch discriminative subsets from Huggingface with retry logic"""
+        def _fetch():
+            logger.info("Fetching discriminative subsets from Huggingface...")
+            
+            # Load each split separately since they're organized by subset
+            subsets = {}
+            subset_names = ['frontier', 'challenging', 'hard', 'multifile']
+            
+            for subset_name in subset_names:
+                dataset = load_dataset("jatinganhotra/SWE-bench_Verified-discriminative", split=subset_name)
+                subsets[subset_name] = [item['instance_id'] for item in dataset]
+            
+            logger.info(f"Loaded subsets: frontier={len(subsets['frontier'])}, "
+                       f"challenging={len(subsets['challenging'])}, "
+                       f"hard={len(subsets['hard'])}, "
+                       f"multifile={len(subsets['multifile'])}")
+            
+            return subsets
+        
+        # Try to load from cache first
+        if self.subsets_cache.exists():
+            logger.info("Loading discriminative subsets from cache...")
+            with open(self.subsets_cache, 'r') as f:
+                return json.load(f)
+        
+        # Fetch with retry logic
+        subsets = self.fetch_with_retry(_fetch)
+        
+        # Cache the results
+        with open(self.subsets_cache, 'w') as f:
+            json.dump(subsets, f, indent=2)
+        
+        return subsets
+    
+    def scan_experiments_directory(self) -> List[Tuple[str, str, List[str]]]:
+        """Scan experiments directory for submission data"""
+        submissions = []
+        
+        if not self.experiments_dir.exists():
+            logger.error(f"Experiments directory not found: {self.experiments_dir}")
+            return submissions
+        
+        for submission_dir in self.experiments_dir.iterdir():
+            if not submission_dir.is_dir():
+                continue
+                
+            results_file = submission_dir / "results" / "results.json"
+            if not results_file.exists():
+                logger.warning(f"No results.json found for {submission_dir.name}")
+                continue
+            
+            try:
+                with open(results_file, 'r') as f:
+                    results = json.load(f)
+                
+                resolved_instances = results.get('resolved', [])
+                submissions.append((submission_dir.name, submission_dir.name, resolved_instances))
+                
+            except Exception as e:
+                logger.warning(f"Error processing {submission_dir.name}: {e}")
+                continue
+        
+        logger.info(f"Found {len(submissions)} valid submissions")
+        return submissions
+    
+    def calculate_subset_performance(self, resolved_instances: List[str], subsets: Dict) -> Dict:
+        """Calculate performance across different subsets"""
+        performance = {}
+        
+        for subset_name, subset_instances in subsets.items():
+            resolved_in_subset = [inst for inst in resolved_instances if inst in subset_instances]
+            total_in_subset = len(subset_instances)
+            solved_count = len(resolved_in_subset)
+            percentage = (solved_count / total_in_subset * 100) if total_in_subset > 0 else 0.0
+            
+            performance[subset_name] = {
+                'solved': solved_count,
+                'total': total_in_subset,
+                'percentage': round(percentage, 1)
+            }
+        
+        # Calculate full benchmark performance
+        total_resolved = len(resolved_instances)
+        full_percentage = (total_resolved / 500) * 100
+        performance['full'] = {
+            'solved': total_resolved,
+            'total': 500,
+            'percentage': round(full_percentage, 1)
+        }
+        
+        return performance
+    
+    def generate_verified_models_data(self) -> Tuple[List[Dict], Dict]:
+        """Generate VERIFIED_MODELS and VERIFIED_MODELS_MAPPING data"""
+        
+        # Check cache first
+        if self.models_cache.exists() and self.mapping_cache.exists():
+            logger.info("Loading models data from cache...")
+            with open(self.models_cache, 'r') as f:
+                models = json.load(f)
+            with open(self.mapping_cache, 'r') as f:
+                mapping = json.load(f)
+            return models, mapping
+        
+        logger.info("Generating fresh models data...")
+        
+        # Get official leaderboard mapping
+        from src.data.leaderboard_scraper import get_official_leaderboard_mapping
+        official_mapping = get_official_leaderboard_mapping(self.experiments_dir)
+        
+        # Fetch discriminative subsets
+        subsets = self.fetch_discriminative_subsets()
+        
+        # Scan experiments directory
+        submissions = self.scan_experiments_directory()
+        
+        verified_models = []
+        verified_models_mapping = {}
+        
+        for submission_id, agent_name, resolved_instances in submissions:
+            # Get official agent name and date from mapping
+            official_data = official_mapping.get(submission_id, {})
+            official_agent_name = official_data.get('official_agent_name', agent_name)
+            official_date = official_data.get('official_date', 'not yet published on leaderboard')
+            
+            # Calculate performance across subsets
+            performance = self.calculate_subset_performance(resolved_instances, subsets)
+            
+            # Create model entry
+            model_entry = {
+                'agent_name': official_agent_name,
+                'submission_id': submission_id,
+                'submission_date': official_date,
+                'rank': 0,  # Will be calculated later
+                'full_solved': performance['full']['solved'],
+                'full_total': performance['full']['total'],
+                'full_percentage': performance['full']['percentage'],
+            }
+            
+            # Add subset performance
+            for subset_name in ['frontier', 'challenging', 'hard', 'multifile']:
+                if subset_name in performance:
+                    model_entry.update({
+                        f'{subset_name}_solved': performance[subset_name]['solved'],
+                        f'{subset_name}_total': performance[subset_name]['total'],
+                        f'{subset_name}_percentage': performance[subset_name]['percentage']
+                    })
+            
+            verified_models.append(model_entry)
+            
+            # Create mapping entry using submission_id as key for uniqueness
+            verified_models_mapping[submission_id] = {
+                'agent_name': official_agent_name,
+                'submission_date': official_date,
+                'resolved_instances': resolved_instances
+            }
+        
+        # Sort by full benchmark performance and assign ranks
+        verified_models.sort(key=lambda x: x['full_percentage'], reverse=True)
+        for i, model in enumerate(verified_models):
+            model['rank'] = i + 1
+        
+        # Cache the results
+        with open(self.models_cache, 'w') as f:
+            json.dump(verified_models, f, indent=2)
+        with open(self.mapping_cache, 'w') as f:
+            json.dump(verified_models_mapping, f, indent=2)
+        
+        logger.info(f"Generated data for {len(verified_models)} models")
+        return verified_models, verified_models_mapping
+    
+    def get_cached_data(self) -> Optional[Tuple[List[Dict], Dict]]:
+        """Get cached data if available"""
+        if self.models_cache.exists() and self.mapping_cache.exists():
+            try:
+                with open(self.models_cache, 'r') as f:
+                    models = json.load(f)
+                with open(self.mapping_cache, 'r') as f:
+                    mapping = json.load(f)
+                return models, mapping
+            except Exception as e:
+                logger.warning(f"Error loading cached data: {e}")
+        return None
+    
+    def refresh_cache(self):
+        """Force refresh of cached data"""
+        logger.info("Refreshing cache...")
+        
+        # Remove cache files including directory mapping
+        directory_mapping_cache = self.cache_dir / "directory_mapping.json"
+        for cache_file in [self.models_cache, self.mapping_cache, self.subsets_cache, directory_mapping_cache]:
+            if cache_file.exists():
+                cache_file.unlink()
+        
+        # Regenerate data
+        return self.generate_verified_models_data()
+
+# Global instance for easy access
+_fetcher = None
+
+def get_leaderboard_data() -> Tuple[List[Dict], Dict]:
+    """Get leaderboard data (cached or fresh)"""
+    global _fetcher
+    if _fetcher is None:
+        _fetcher = LeaderboardDataFetcher()
+    
+    # Try to get cached data first
+    cached_data = _fetcher.get_cached_data()
+    if cached_data is not None:
+        return cached_data
+    
+    # Generate fresh data
+    return _fetcher.generate_verified_models_data()
+
+def refresh_leaderboard_data() -> Tuple[List[Dict], Dict]:
+    """Force refresh of leaderboard data"""
+    global _fetcher
+    if _fetcher is None:
+        _fetcher = LeaderboardDataFetcher()
+    
+    return _fetcher.refresh_cache()
+
+if __name__ == "__main__":
+    # Test the data fetcher
+    fetcher = LeaderboardDataFetcher()
+    models, mapping = fetcher.generate_verified_models_data()
+    
+    print(f"Generated data for {len(models)} models")
+    print("Top 5 models:")
+    for i, model in enumerate(models[:5]):
+        print(f"{i+1}. {model['agent_name']}: {model['full_percentage']}% "
+              f"(frontier: {model.get('frontier_percentage', 0)}%)")

src/data/leaderboard_scraper.py

@@ -0,0 +1,99 @@
+import json
+import re
+import logging
+from pathlib import Path
+from typing import Dict, Optional
+from datetime import datetime
+import yaml
+
+logger = logging.getLogger(__name__)
+
+class SWEBenchMetadataReader:
+    """Reads metadata.yaml files from experiment directories"""
+    
+    def __init__(self, cache_dir: str = "leaderboard_cache"):
+        self.cache_dir = Path(cache_dir)
+        self.cache_dir.mkdir(exist_ok=True)
+    
+    def _extract_date_from_dirname(self, dirname: str) -> str:
+        """Extract date from directory name format YYYYMMDD_*"""
+        match = re.match(r'(\d{8})_', dirname)
+        if match:
+            date_str = match.group(1)
+            try:
+                # Convert YYYYMMDD to readable format
+                date_obj = datetime.strptime(date_str, '%Y%m%d')
+                return date_obj.strftime('%Y-%m-%d')
+            except ValueError:
+                return date_str
+        return 'not yet published on leaderboard'
+    
+    def get_cached_mapping(self) -> Optional[Dict]:
+        """Get cached directory mapping if available"""
+        mapping_file = self.cache_dir / "directory_mapping.json"
+        if mapping_file.exists():
+            try:
+                with open(mapping_file, 'r') as f:
+                    return json.load(f)
+            except Exception as e:
+                logger.warning(f"Error loading cached mapping: {e}")
+        return None
+    
+    def save_mapping(self, mapping: Dict):
+        """Save directory mapping to cache"""
+        mapping_file = self.cache_dir / "directory_mapping.json"
+        with open(mapping_file, 'w') as f:
+            json.dump(mapping, f, indent=2)
+        logger.info(f"Saved directory mapping to {mapping_file}")
+
+def get_official_leaderboard_mapping(experiments_dir: Path) -> Dict:
+    """Get mapping between directory names and official agent data from metadata.yaml files"""
+    metadata_reader = SWEBenchMetadataReader()
+    
+    # Check if we have cached mapping
+    cached_mapping = metadata_reader.get_cached_mapping()
+    if cached_mapping:
+        logger.info("Using cached directory mapping")
+        return cached_mapping
+    
+    logger.info("Generating mapping from metadata.yaml files...")
+    mapping = {}
+    
+    # Create mapping for all experiment directories using metadata.yaml
+    if experiments_dir.exists():
+        for exp_dir in experiments_dir.iterdir():
+            if exp_dir.is_dir():
+                dir_date = metadata_reader._extract_date_from_dirname(exp_dir.name)
+                metadata_file = exp_dir / "metadata.yaml"
+                
+                # Try to read official name from metadata.yaml
+                official_name = exp_dir.name  # Default fallback
+                if metadata_file.exists():
+                    try:
+                        with open(metadata_file, 'r', encoding='utf-8') as f:
+                            metadata = yaml.safe_load(f)
+                        
+                        # Extract official name from info.name field
+                        if metadata and 'info' in metadata and 'name' in metadata['info']:
+                            official_name = metadata['info']['name']
+                            logger.debug(f"Found official name for {exp_dir.name}: {official_name}")
+                        else:
+                            logger.warning(f"No info.name found in {metadata_file}")
+                    except Exception as e:
+                        logger.warning(f"Error reading {metadata_file}: {e}")
+                else:
+                    logger.warning(f"No metadata.yaml found for {exp_dir.name}")
+                
+                mapping[exp_dir.name] = {
+                    'official_agent_name': official_name,
+                    'official_date': dir_date,
+                    'directory_date': dir_date,
+                    'directory_name': exp_dir.name,
+                    'score': '',
+                    'matched': official_name != exp_dir.name  # True if we found a different name in metadata
+                }
+    
+    logger.info(f"Generated mapping for {len(mapping)} directories from metadata.yaml files")
+    metadata_reader.save_mapping(mapping)
+    
+    return mapping

src/utils/automated_updater.py

@@ -0,0 +1,305 @@
+#!/usr/bin/env python3
+"""
+Automated updater for SWE-Bench leaderboard data.
+Handles daily updates, new submission detection, and semi-automated name mapping.
+"""
+
+import os
+import json
+import subprocess
+import tempfile
+import shutil
+import logging
+import time
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple, Set
+
+from src.data.leaderboard_scraper import SWEBenchMetadataReader
+from src.data.leaderboard_data_fetcher import LeaderboardDataFetcher
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
+
+class AutomatedUpdater:
+    """Automated updater for SWE-Bench leaderboard data"""
+    
+    def __init__(self, cache_dir: str = "leaderboard_cache"):
+        self.cache_dir = Path(cache_dir)
+        self.cache_dir.mkdir(exist_ok=True)
+        
+        # Status files
+        self.last_update_file = self.cache_dir / "last_update.json"
+        self.update_status_file = self.cache_dir / "update_status.json"
+        self.new_submissions_file = self.cache_dir / "new_submissions.json"
+        
+        # Git repo details
+        self.experiments_repo_url = "https://github.com/SWE-bench/experiments.git"
+        self.experiments_dir = Path("leaderboard_swe_bench_experiments")
+        
+        # Components
+        self.metadata_reader = SWEBenchMetadataReader(cache_dir)
+        self.fetcher = LeaderboardDataFetcher(cache_dir)
+        
+    def get_last_update_info(self) -> Dict:
+        """Get information about the last update"""
+        if self.last_update_file.exists():
+            try:
+                with open(self.last_update_file, 'r') as f:
+                    return json.load(f)
+            except Exception as e:
+                logger.warning(f"Error loading last update info: {e}")
+        
+        return {
+            "last_update_time": None,
+            "last_submission_count": 0,
+            "last_successful_update": None,
+            "update_status": "never_updated"
+        }
+    
+    def save_last_update_info(self, info: Dict):
+        """Save information about the last update"""
+        with open(self.last_update_file, 'w') as f:
+            json.dump(info, f, indent=2)
+    
+    def get_update_status(self) -> Dict:
+        """Get current update status for UI display"""
+        if self.update_status_file.exists():
+            try:
+                with open(self.update_status_file, 'r') as f:
+                    return json.load(f)
+            except Exception as e:
+                logger.warning(f"Error loading update status: {e}")
+        
+        return {
+            "status": "unknown",
+            "message": "Update status not available",
+            "last_checked": None,
+            "new_submissions_count": 0
+        }
+    
+    def save_update_status(self, status: Dict):
+        """Save update status for UI display"""
+        status["last_checked"] = datetime.now().isoformat()
+        with open(self.update_status_file, 'w') as f:
+            json.dump(status, f, indent=2)
+    
+    def clone_or_update_experiments_repo(self) -> bool:
+        """Clone or update the experiments repository"""
+        try:
+            if self.experiments_dir.exists():
+                logger.info("Updating existing experiments repository...")
+                result = subprocess.run(
+                    ["git", "pull"], 
+                    cwd=self.experiments_dir,
+                    capture_output=True,
+                    text=True,
+                    timeout=300
+                )
+                if result.returncode != 0:
+                    logger.error(f"Git pull failed: {result.stderr}")
+                    return False
+            else:
+                logger.info("Cloning experiments repository...")
+                result = subprocess.run(
+                    ["git", "clone", self.experiments_repo_url, str(self.experiments_dir)],
+                    capture_output=True,
+                    text=True,
+                    timeout=600
+                )
+                if result.returncode != 0:
+                    logger.error(f"Git clone failed: {result.stderr}")
+                    return False
+            
+            logger.info("Successfully updated experiments repository")
+            return True
+            
+        except subprocess.TimeoutExpired:
+            logger.error("Git operation timed out")
+            return False
+        except Exception as e:
+            logger.error(f"Error updating experiments repository: {e}")
+            return False
+    
+    def detect_new_submissions(self) -> Tuple[List[str], int]:
+        """Detect new submissions by comparing current directory list with cached list"""
+        verified_dir = self.experiments_dir / "evaluation" / "verified"
+        
+        if not verified_dir.exists():
+            logger.error(f"Verified directory not found: {verified_dir}")
+            return [], 0
+        
+        # Get current submission directories
+        current_submissions = set()
+        for submission_dir in verified_dir.iterdir():
+            if submission_dir.is_dir():
+                results_file = submission_dir / "results" / "results.json"
+                if results_file.exists():
+                    current_submissions.add(submission_dir.name)
+        
+        # Load previous submission list
+        previous_submissions = set()
+        if self.new_submissions_file.exists():
+            try:
+                with open(self.new_submissions_file, 'r') as f:
+                    data = json.load(f)
+                    previous_submissions = set(data.get("submissions", []))
+            except Exception as e:
+                logger.warning(f"Error loading previous submissions: {e}")
+        
+        # Find new submissions
+        new_submissions = list(current_submissions - previous_submissions)
+        
+        # Save current submissions for next time
+        submission_data = {
+            "submissions": list(current_submissions),
+            "last_check": datetime.now().isoformat(),
+            "new_since_last_check": new_submissions
+        }
+        
+        with open(self.new_submissions_file, 'w') as f:
+            json.dump(submission_data, f, indent=2)
+        
+        logger.info(f"Detected {len(new_submissions)} new submissions out of {len(current_submissions)} total")
+        return new_submissions, len(current_submissions)
+    
+    
+    def process_new_submissions(self, new_submissions: List[str]) -> Dict:
+        """Process new submissions and create mapping entries"""
+        if not new_submissions:
+            return {"processed": 0, "auto_matched": 0}
+        
+        logger.info(f"Processing {len(new_submissions)} new submissions...")
+        
+        # All submissions are automatically processed since we read names from metadata.yaml
+        auto_matched = []
+        
+        for submission_id in new_submissions:
+            logger.info(f"Processing new submission: {submission_id}")
+            
+            auto_matched.append({
+                "submission_id": submission_id,
+                "matched_name": "Will be read from metadata.yaml",
+                "confidence": 1.0
+            })
+        
+        return {
+            "processed": len(new_submissions),
+            "auto_matched": len(auto_matched),
+            "auto_matched_details": auto_matched
+        }
+    
+    def run_daily_update(self) -> Dict:
+        """Run the daily update process"""
+        logger.info("Starting daily update process...")
+        
+        update_result = {
+            "success": False,
+            "timestamp": datetime.now().isoformat(),
+            "message": "",
+            "new_submissions_count": 0,
+            "error": None
+        }
+        
+        try:
+            # Step 1: Update git repository
+            logger.info("Step 1: Updating experiments repository...")
+            if not self.clone_or_update_experiments_repo():
+                raise Exception("Failed to update experiments repository")
+            
+            # Step 2: Detect new submissions
+            logger.info("Step 2: Detecting new submissions...")
+            new_submissions, total_submissions = self.detect_new_submissions()
+            update_result["new_submissions_count"] = len(new_submissions)
+            
+            if new_submissions:
+                logger.info(f"Found {len(new_submissions)} new submissions: {new_submissions}")
+                
+                # Step 3: Process new submissions
+                logger.info("Step 3: Processing new submissions...")
+                processing_result = self.process_new_submissions(new_submissions)
+                
+                update_result["message"] = f"Found {len(new_submissions)} new submissions. All automatically processed with official names from metadata."
+                
+            else:
+                update_result["message"] = f"No new submissions found. Total: {total_submissions} submissions."
+            
+            # Step 4: Refresh leaderboard data cache
+            logger.info("Step 4: Refreshing leaderboard data cache...")
+            try:
+                self.fetcher.refresh_cache()
+                logger.info("Successfully refreshed leaderboard data cache")
+            except Exception as e:
+                logger.warning(f"Error refreshing cache: {e}")
+                # Continue anyway - this is not critical for detecting new submissions
+            
+            update_result["success"] = True
+            
+        except Exception as e:
+            logger.error(f"Daily update failed: {e}")
+            update_result["error"] = str(e)
+            update_result["message"] = f"Update failed: {e}"
+        
+        # Save update status
+        self.save_update_status(update_result)
+        
+        # Update last update info
+        last_update_info = self.get_last_update_info()
+        last_update_info.update({
+            "last_update_time": update_result["timestamp"],
+            "update_status": "success" if update_result["success"] else "failed",
+            "last_error": update_result.get("error")
+        })
+        
+        if update_result["success"]:
+            last_update_info["last_successful_update"] = update_result["timestamp"]
+        
+        self.save_last_update_info(last_update_info)
+        
+        logger.info(f"Daily update completed. Success: {update_result['success']}")
+        return update_result
+    
+    def should_run_update(self) -> bool:
+        """Check if an update should be run based on timing"""
+        last_update_info = self.get_last_update_info()
+        
+        if not last_update_info.get("last_update_time"):
+            return True  # Never updated
+        
+        try:
+            last_update = datetime.fromisoformat(last_update_info["last_update_time"])
+            time_since_update = datetime.now() - last_update
+            
+            # Run update if more than 24 hours since last update
+            return time_since_update > timedelta(hours=24)
+            
+        except ValueError:
+            return True  # Invalid timestamp, run update
+    
+
+def run_automated_update() -> Dict:
+    """Entry point for running automated update"""
+    updater = AutomatedUpdater()
+    return updater.run_daily_update()
+
+def get_update_status() -> Dict:
+    """Get current update status for UI"""
+    updater = AutomatedUpdater()
+    return updater.get_update_status()
+
+def get_last_update_info() -> Dict:
+    """Get last update information for UI"""
+    updater = AutomatedUpdater()
+    return updater.get_last_update_info()
+
+def should_show_update_notification() -> bool:
+    """Check if update notification should be shown in UI"""
+    updater = AutomatedUpdater()
+    status = updater.get_update_status()
+    return status.get("new_submissions_count", 0) > 0
+
+if __name__ == "__main__":
+    # Run automated update
+    result = run_automated_update()
+    print(json.dumps(result, indent=2))

src/utils/export_utils.py

@@ -0,0 +1,286 @@
+"""
+Export utilities for downloading leaderboard data in various formats
+"""
+
+import pandas as pd
+import json
+import csv
+import io
+from datetime import datetime
+from typing import List, Dict, Any, Tuple
+
+
+def format_export_data(performance_data: List[Dict[str, Any]], include_metadata: bool = True) -> List[Dict[str, Any]]:
+    """
+    Format performance data for export with clean column names and optional metadata
+    """
+    export_data = []
+    
+    for row in performance_data:
+        export_row = {
+            'rank': row['rank'],
+            'agent_name': row['agent_name'],
+            'submission_id': row['submission_id'],
+            'submission_date': row.get('submission_date', 'not yet published on leaderboard'),
+            
+            # Full benchmark
+            'full_solved': row['full_solved'],
+            'full_total': row['full_total'],
+            'full_percentage': row['full_percentage'],
+            
+            # Frontier subset
+            'frontier_solved': row['frontier_solved'],
+            'frontier_total': row['frontier_total'],
+            'frontier_percentage': row['frontier_percentage'],
+            
+            # Challenging subset
+            'challenging_solved': row['challenging_solved'],
+            'challenging_total': row['challenging_total'],
+            'challenging_percentage': row['challenging_percentage'],
+            
+            # Hard subset
+            'hard_solved': row['hard_solved'],
+            'hard_total': row['hard_total'],
+            'hard_percentage': row['hard_percentage'],
+            
+            # MultiFile subset
+            'multifile_solved': row['multifile_solved'],
+            'multifile_total': row['multifile_total'],
+            'multifile_percentage': row['multifile_percentage'],
+        }
+        
+        export_data.append(export_row)
+    
+    return export_data
+
+
+def create_csv_export(performance_data: List[Dict[str, Any]], filename_prefix: str = "swe_bench_leaderboard") -> Tuple[str, str]:
+    """
+    Create CSV export from performance data
+    Returns: (csv_content, filename)
+    """
+    export_data = format_export_data(performance_data)
+    
+    # Create CSV content
+    output = io.StringIO()
+    if export_data:
+        fieldnames = export_data[0].keys()
+        writer = csv.DictWriter(output, fieldnames=fieldnames)
+        writer.writeheader()
+        writer.writerows(export_data)
+    
+    csv_content = output.getvalue()
+    output.close()
+    
+    # Generate filename with timestamp
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    filename = f"{filename_prefix}_{timestamp}.csv"
+    
+    return csv_content, filename
+
+
+def create_json_export(performance_data: List[Dict[str, Any]], filename_prefix: str = "swe_bench_leaderboard") -> Tuple[str, str]:
+    """
+    Create JSON export from performance data
+    Returns: (json_content, filename)
+    """
+    export_data = format_export_data(performance_data)
+    
+    # Create export metadata
+    export_metadata = {
+        "export_timestamp": datetime.now().isoformat(),
+        "total_agents": len(export_data),
+        "data_description": "SWE-Bench Verified Discriminative Subsets Leaderboard Data",
+        "subsets": {
+            "full": {"name": "SWE-Bench Verified", "total_instances": 500},
+            "frontier": {"name": "Frontier Subset", "total_instances": 95, "description": "Problems solved by ≤5 agents"},
+            "challenging": {"name": "Challenging Subset", "total_instances": 155, "description": "Problems solved by ≤20 agents"},
+            "hard": {"name": "Hard Subset", "total_instances": 45, "description": "All Hard difficulty problems"},
+            "multifile": {"name": "MultiFile Subset", "total_instances": 40, "description": "Multi-file problems solved by ≤10 agents"}
+        }
+    }
+    
+    # Combine metadata and data
+    full_export = {
+        "metadata": export_metadata,
+        "leaderboard_data": export_data
+    }
+    
+    json_content = json.dumps(full_export, indent=2, ensure_ascii=False)
+    
+    # Generate filename with timestamp
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    filename = f"{filename_prefix}_{timestamp}.json"
+    
+    return json_content, filename
+
+
+def create_subset_export(performance_data: List[Dict[str, Any]], subset_name: str, format_type: str = "csv") -> Tuple[str, str]:
+    """
+    Create export for a specific subset with relevant columns only
+    """
+    if subset_name == "full":
+        subset_columns = ['rank', 'agent_name', 'submission_date', 'full_solved', 'full_total', 'full_percentage']
+        filename_prefix = "swe_bench_full_benchmark"
+    elif subset_name == "frontier":
+        subset_columns = ['rank', 'agent_name', 'submission_date', 'frontier_solved', 'frontier_total', 'frontier_percentage']
+        filename_prefix = "swe_bench_frontier_subset"
+    elif subset_name == "challenging":
+        subset_columns = ['rank', 'agent_name', 'submission_date', 'challenging_solved', 'challenging_total', 'challenging_percentage']
+        filename_prefix = "swe_bench_challenging_subset"
+    elif subset_name == "hard":
+        subset_columns = ['rank', 'agent_name', 'submission_date', 'hard_solved', 'hard_total', 'hard_percentage']
+        filename_prefix = "swe_bench_hard_subset"
+    elif subset_name == "multifile":
+        subset_columns = ['rank', 'agent_name', 'submission_date', 'multifile_solved', 'multifile_total', 'multifile_percentage']
+        filename_prefix = "swe_bench_multifile_subset"
+    else:
+        raise ValueError(f"Unknown subset: {subset_name}")
+    
+    # Filter data to only include relevant columns
+    export_data = format_export_data(performance_data)
+    filtered_data = []
+    
+    for row in export_data:
+        filtered_row = {col: row[col] for col in subset_columns if col in row}
+        filtered_data.append(filtered_row)
+    
+    if format_type.lower() == "csv":
+        # Create CSV content
+        output = io.StringIO()
+        if filtered_data:
+            fieldnames = filtered_data[0].keys()
+            writer = csv.DictWriter(output, fieldnames=fieldnames)
+            writer.writeheader()
+            writer.writerows(filtered_data)
+        
+        content = output.getvalue()
+        output.close()
+        
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        filename = f"{filename_prefix}_{timestamp}.csv"
+        
+    elif format_type.lower() == "json":
+        # Create JSON content with metadata
+        subset_metadata = {
+            "export_timestamp": datetime.now().isoformat(),
+            "subset": subset_name,
+            "total_agents": len(filtered_data),
+            "description": f"SWE-Bench {subset_name.title()} Subset Results"
+        }
+        
+        full_export = {
+            "metadata": subset_metadata,
+            "results": filtered_data
+        }
+        
+        content = json.dumps(full_export, indent=2, ensure_ascii=False)
+        
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        filename = f"{filename_prefix}_{timestamp}.json"
+    
+    else:
+        raise ValueError(f"Unsupported format: {format_type}")
+    
+    return content, filename
+
+
+def create_comparison_export(performance_data: List[Dict[str, Any]], agents_to_compare: List[str]) -> Tuple[str, str]:
+    """
+    Create export comparing specific agents across all subsets
+    """
+    # Filter data for selected agents
+    comparison_data = [row for row in performance_data if row['agent_name'] in agents_to_compare]
+    
+    # Create comparison-focused format
+    comparison_rows = []
+    for row in comparison_data:
+        comparison_row = {
+            'agent_name': row['agent_name'],
+            'submission_date': row.get('submission_date', 'not yet published on leaderboard'),
+            'full_percentage': row['full_percentage'],
+            'frontier_percentage': row['frontier_percentage'],
+            'challenging_percentage': row['challenging_percentage'],
+            'hard_percentage': row['hard_percentage'],
+            'multifile_percentage': row['multifile_percentage'],
+            'frontier_vs_full_drop': round(row['full_percentage'] - row['frontier_percentage'], 1),
+            'challenging_vs_full_drop': round(row['full_percentage'] - row['challenging_percentage'], 1),
+        }
+        comparison_rows.append(comparison_row)
+    
+    # Sort by full benchmark performance
+    comparison_rows.sort(key=lambda x: x['full_percentage'], reverse=True)
+    
+    # Create JSON export
+    export_data = {
+        "metadata": {
+            "export_timestamp": datetime.now().isoformat(),
+            "comparison_type": "agent_comparison",
+            "agents_compared": agents_to_compare,
+            "total_agents": len(comparison_rows)
+        },
+        "comparison_data": comparison_rows
+    }
+    
+    content = json.dumps(export_data, indent=2, ensure_ascii=False)
+    
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    filename = f"swe_bench_agent_comparison_{timestamp}.json"
+    
+    return content, filename
+
+
+# Gradio-compatible export functions
+import tempfile
+import os
+
+def export_full_leaderboard_csv(performance_data: List[Dict[str, Any]]) -> str:
+    """Gradio-compatible function to export full leaderboard as CSV"""
+    content, filename = create_csv_export(performance_data, "swe_bench_full_leaderboard")
+    
+    # Write to system temp directory for Gradio download
+    temp_dir = tempfile.gettempdir()
+    temp_path = os.path.join(temp_dir, filename)
+    with open(temp_path, 'w', encoding='utf-8') as f:
+        f.write(content)
+    
+    return temp_path
+
+
+def export_full_leaderboard_json(performance_data: List[Dict[str, Any]]) -> str:
+    """Gradio-compatible function to export full leaderboard as JSON"""
+    content, filename = create_json_export(performance_data, "swe_bench_full_leaderboard")
+    
+    # Write to system temp directory for Gradio download
+    temp_dir = tempfile.gettempdir()
+    temp_path = os.path.join(temp_dir, filename)
+    with open(temp_path, 'w', encoding='utf-8') as f:
+        f.write(content)
+    
+    return temp_path
+
+
+def export_subset_csv(performance_data: List[Dict[str, Any]], subset_name: str) -> str:
+    """Gradio-compatible function to export specific subset as CSV"""
+    content, filename = create_subset_export(performance_data, subset_name, "csv")
+    
+    # Write to system temp directory for Gradio download
+    temp_dir = tempfile.gettempdir()
+    temp_path = os.path.join(temp_dir, filename)
+    with open(temp_path, 'w', encoding='utf-8') as f:
+        f.write(content)
+    
+    return temp_path
+
+
+def export_subset_json(performance_data: List[Dict[str, Any]], subset_name: str) -> str:
+    """Gradio-compatible function to export specific subset as JSON"""
+    content, filename = create_subset_export(performance_data, subset_name, "json")
+    
+    # Write to system temp directory for Gradio download
+    temp_dir = tempfile.gettempdir()
+    temp_path = os.path.join(temp_dir, filename)
+    with open(temp_path, 'w', encoding='utf-8') as f:
+        f.write(content)
+    
+    return temp_path

src/utils/update_scheduler.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+"""
+Scheduler for automated SWE-Bench leaderboard updates.
+Can be run as a standalone script or integrated into the Gradio app.
+"""
+
+import time
+import schedule
+import threading
+import logging
+from datetime import datetime, timedelta
+from pathlib import Path
+import json
+
+from src.utils.automated_updater import AutomatedUpdater
+
+logger = logging.getLogger(__name__)
+
+class UpdateScheduler:
+    """Scheduler for automated leaderboard updates"""
+    
+    def __init__(self):
+        self.updater = AutomatedUpdater()
+        self.running = False
+        self.scheduler_thread = None
+        
+    def run_scheduled_update(self):
+        """Run the scheduled update with error handling"""
+        try:
+            logger.info("Running scheduled update...")
+            result = self.updater.run_daily_update()
+            
+            if result["success"]:
+                logger.info(f"Scheduled update completed successfully. {result['message']}")
+            else:
+                logger.error(f"Scheduled update failed: {result['message']}")
+                
+        except Exception as e:
+            logger.error(f"Error in scheduled update: {e}")
+    
+    def start_scheduler(self):
+        """Start the update scheduler"""
+        if self.running:
+            logger.warning("Scheduler is already running")
+            return
+        
+        # Schedule daily updates at 6 AM UTC
+        schedule.every().day.at("06:00").do(self.run_scheduled_update)
+        
+        # Also run immediately if it's been more than 24 hours
+        if self.updater.should_run_update():
+            logger.info("Running immediate update (more than 24 hours since last update)")
+            schedule.every().minute.do(self._run_immediate_update).tag('immediate')
+        
+        self.running = True
+        self.scheduler_thread = threading.Thread(target=self._scheduler_loop, daemon=True)
+        self.scheduler_thread.start()
+        
+        logger.info("Update scheduler started")
+    
+    def _run_immediate_update(self):
+        """Run immediate update and remove the job"""
+        self.run_scheduled_update()
+        schedule.clear('immediate')  # Remove the immediate update job
+        return schedule.CancelJob
+    
+    def stop_scheduler(self):
+        """Stop the update scheduler"""
+        if not self.running:
+            return
+        
+        self.running = False
+        schedule.clear()
+        
+        if self.scheduler_thread and self.scheduler_thread.is_alive():
+            self.scheduler_thread.join(timeout=5)
+        
+        logger.info("Update scheduler stopped")
+    
+    def _scheduler_loop(self):
+        """Main scheduler loop"""
+        while self.running:
+            try:
+                schedule.run_pending()
+                time.sleep(60)  # Check every minute
+            except Exception as e:
+                logger.error(f"Error in scheduler loop: {e}")
+                time.sleep(60)
+    
+    def force_update(self):
+        """Force an immediate update"""
+        logger.info("Forcing immediate update...")
+        return self.updater.run_daily_update()
+    
+    def get_status(self):
+        """Get scheduler status"""
+        return {
+            "running": self.running,
+            "next_run": str(schedule.next_run()) if schedule.jobs else None,
+            "job_count": len(schedule.jobs)
+        }
+
+# Global scheduler instance
+_scheduler = None
+
+def get_scheduler() -> UpdateScheduler:
+    """Get the global scheduler instance"""
+    global _scheduler
+    if _scheduler is None:
+        _scheduler = UpdateScheduler()
+    return _scheduler
+
+def start_automated_updates():
+    """Start automated updates"""
+    scheduler = get_scheduler()
+    scheduler.start_scheduler()
+
+def stop_automated_updates():
+    """Stop automated updates"""
+    scheduler = get_scheduler()
+    scheduler.stop_scheduler()
+
+def force_update():
+    """Force an immediate update"""
+    scheduler = get_scheduler()
+    return scheduler.force_update()
+
+def get_scheduler_status():
+    """Get scheduler status"""
+    scheduler = get_scheduler()
+    return scheduler.get_status()
+
+if __name__ == "__main__":
+    # Run as standalone scheduler
+    logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+    
+    scheduler = UpdateScheduler()
+    scheduler.start_scheduler()
+    
+    try:
+        # Keep running
+        while True:
+            time.sleep(60)
+    except KeyboardInterrupt:
+        logger.info("Received interrupt signal")
+        scheduler.stop_scheduler()