handle slow charts

app.py

@@ -19,6 +19,7 @@ import io
 import base64
 from itertools import combinations
 import warnings
+import time
 warnings.filterwarnings('ignore')
 
 # Configure page
@@ -64,7 +65,19 @@ def load_comprehensive_data():
         df = pd.read_csv("comprehensive_benchmark_scores.csv", index_col=0, encoding='utf-8')
         
         # Clean the data - handle list-like values stored as strings
-        for col in df.columns:
+        # Process in batches for better performance with large datasets
+        total_cols = len(df.columns)
+        
+        if total_cols > 20:
+            # Show progress for large datasets
+            progress_text = st.empty()
+            progress_bar = st.progress(0)
+            
+        for i, col in enumerate(df.columns):
+            if total_cols > 20:
+                progress_text.text(f"Processing column {i+1}/{total_cols}: {col}")
+                progress_bar.progress((i+1) / total_cols)
+                
             def extract_value(x):
                 if pd.isna(x):
                     return np.nan
@@ -85,6 +98,10 @@ def load_comprehensive_data():
             df[col] = df[col].apply(extract_value)
             df[col] = pd.to_numeric(df[col], errors='coerce')
         
+        if total_cols > 20:
+            progress_text.empty()
+            progress_bar.empty()
+        
         # Filter to only models that have data for at least a few benchmarks
         min_benchmarks = 3
         df_filtered = df.dropna(thresh=min_benchmarks, axis=0)
@@ -334,6 +351,7 @@ def filter_target_benchmarks(df):
     
     return df[available_benchmarks].copy()
 
+@st.cache_data
 def estimate_missing_ranks(df, method='spearman', min_corr=0.3, min_benchmarks=3):
     """
     Estimate missing benchmark ranks using rank correlation-based imputation.
@@ -351,7 +369,7 @@ def estimate_missing_ranks(df, method='spearman', min_corr=0.3, min_benchmarks=3
     df_ranks = df.rank(method='min', ascending=False, na_option='keep')
     df_ranks_imputed = df_ranks.copy()
     
-    # Compute rank correlation matrix
+    # Compute rank correlation matrix once
     if method == 'spearman':
         rank_corr_matrix = df_ranks.corr(method='spearman')
     elif method == 'kendall':
@@ -359,23 +377,42 @@ def estimate_missing_ranks(df, method='spearman', min_corr=0.3, min_benchmarks=3
     else:
         rank_corr_matrix = df_ranks.corr(method='pearson')  # fallback
     
+    # Pre-compute correlation thresholds to avoid repeated calculations
+    valid_correlations = {}
+    for benchmark in df.columns:
+        valid_correlations[benchmark] = []
+        for other_bench in df.columns:
+            if benchmark != other_bench:
+                corr_val = rank_corr_matrix.loc[benchmark, other_bench]
+                if not pd.isna(corr_val) and abs(corr_val) >= min_corr:
+                    valid_correlations[benchmark].append((other_bench, abs(corr_val)))
+        # Sort by correlation strength for better prediction
+        valid_correlations[benchmark].sort(key=lambda x: x[1], reverse=True)
+    
     # For each model and benchmark combination with missing data
+    missing_count = 0
+    total_missing = df_ranks.isna().sum().sum()
+    
     for model_idx in df.index:
-        for benchmark in df.columns:
-            if pd.isna(df_ranks.loc[model_idx, benchmark]):
-                # Find benchmarks this model has ranks for
-                available_benchmarks = df_ranks.columns[df_ranks.loc[model_idx].notna()].tolist()
-                
-                if len(available_benchmarks) >= min_benchmarks:
-                    # Get rank correlations between target benchmark and available benchmarks
+        available_benchmarks = df_ranks.columns[df_ranks.loc[model_idx].notna()].tolist()
+        
+        if len(available_benchmarks) >= min_benchmarks:
+            for benchmark in df.columns:
+                if pd.isna(df_ranks.loc[model_idx, benchmark]):
+                    # Get pre-computed valid correlations for this benchmark
+                    valid_pairs = valid_correlations[benchmark]
+                    
                     correlations = []
                     ranks = []
                     
-                    for avail_bench in available_benchmarks:
-                        corr_val = rank_corr_matrix.loc[benchmark, avail_bench]
-                        if not pd.isna(corr_val) and abs(corr_val) >= min_corr:
-                            correlations.append(abs(corr_val))  # Use absolute correlation as weight
-                            ranks.append(df_ranks.loc[model_idx, avail_bench])
+                    for other_bench, corr_strength in valid_pairs:
+                        if other_bench in available_benchmarks:
+                            correlations.append(corr_strength)
+                            ranks.append(df_ranks.loc[model_idx, other_bench])
+                            
+                            # Limit to top 5 most correlated benchmarks for efficiency
+                            if len(correlations) >= 5:
+                                break
                     
                     if len(correlations) > 0:
                         # Weighted average of ranks using correlations as weights
@@ -387,9 +424,11 @@ def estimate_missing_ranks(df, method='spearman', min_corr=0.3, min_benchmarks=3
                         estimated_rank = np.average(ranks, weights=weights)
                         
                         df_ranks_imputed.loc[model_idx, benchmark] = estimated_rank
+                        missing_count += 1
     
     return df_ranks_imputed
 
+@st.cache_data
 def create_consensus_ranking(df, method='spearman', use_rank_imputation=True):
     """
     Create a consensus ranking using rank correlation-based estimation.
@@ -461,6 +500,81 @@ def create_consensus_ranking(df, method='spearman', use_rank_imputation=True):
     
     return ranking_df, df_ranks, metadata
 
+@st.cache_data
+def create_optimized_radar_chart(df_display, selected_models, selected_benchmarks_for_radar):
+    """Create an optimized radar chart for the selected models and benchmarks."""
+    if not selected_benchmarks_for_radar or not selected_models:
+        return None
+    
+    # Pre-filter data to only what we need
+    filtered_data = df_display.loc[selected_models, selected_benchmarks_for_radar]
+    clean_benchmark_names = [clean_benchmark_name(b) for b in selected_benchmarks_for_radar]
+    
+    # Define colors for different models
+    colors_list = ['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd', 
+                  '#8c564b', '#e377c2', '#7f7f7f', '#bcbd22', '#17becf']
+    
+    fig = go.Figure()
+    
+    # Calculate dynamic range for better visualization
+    all_values = filtered_data.values.flatten()
+    all_values = all_values[~pd.isna(all_values)]
+    
+    if len(all_values) > 0:
+        min_val = float(np.min(all_values))
+        max_val = float(np.max(all_values))
+        # Add some padding
+        range_padding = (max_val - min_val) * 0.1
+        radar_min = max(0, min_val - range_padding)
+        radar_max = min(1, max_val + range_padding)
+    else:
+        radar_min, radar_max = 0, 1
+    
+    for i, model in enumerate(selected_models):
+        # Get model data for selected benchmarks only
+        model_scores = []
+        for benchmark in selected_benchmarks_for_radar:
+            score = filtered_data.loc[model, benchmark]
+            # Convert to float, use 0.0 for any remaining NaN values
+            model_scores.append(0.0 if pd.isna(score) else float(score))
+        
+        # Close the radar chart by adding the first value at the end
+        radar_values = model_scores + [model_scores[0]]
+        radar_benchmarks = clean_benchmark_names + [clean_benchmark_names[0]]
+        
+        # Create model name for legend (remove path prefix if present)
+        model_display_name = model.split('/')[-1] if '/' in model else model
+        
+        # Use color from list, cycling if needed
+        model_color = colors_list[i % len(colors_list)]
+        
+        fig.add_trace(go.Scatterpolar(
+            r=radar_values,
+            theta=radar_benchmarks,
+            fill='toself',
+            name=model_display_name,
+            line_color=model_color,
+            hovertemplate='<b>%{theta}</b><br>Score: %{r:.3f}<extra></extra>'
+        ))
+    
+    # Adjust chart size based on number of models
+    chart_height = 600 if len(selected_models) <= 3 else 700
+    
+    fig.update_layout(
+        polar=dict(
+            radialaxis=dict(
+                visible=True,
+                range=[radar_min, radar_max],
+                tickformat='.2f'
+            )),
+        showlegend=True,
+        title=f"Model Performance Radar Chart ({len(selected_benchmarks_for_radar)} benchmarks, {len(selected_models)} models)",
+        width=700,
+        height=chart_height
+    )
+    
+    return fig
+
 def weighted_correlation(x, y, weights):
     """Compute weighted Pearson correlation coefficient."""
     # Remove NaN values
@@ -821,9 +935,11 @@ def main():
     st.markdown('<h1 class="main-header">OpenThoughts Evalchemy Benchmark Explorer</h1>', 
                 unsafe_allow_html=True)
     
-    # Load data
+    # Load data with timing
+    start_time = time.time()
     df = load_comprehensive_data()
     stderr_df = load_stderr_data()
+    load_time = time.time() - start_time
     
     # Debug information (hidden in an expander)
     # with st.expander("🔧 Debug Information", expanded=False):
@@ -907,6 +1023,16 @@ def main():
             valid_benchmarks.append(col)
     df_display = df_display[valid_benchmarks]
     
+    # Performance info
+    st.sidebar.markdown("---")
+    st.sidebar.subheader("⚡ Performance")
+    if load_time > 0:
+        st.sidebar.metric("Data Load Time", f"{load_time:.2f}s")
+    st.sidebar.metric("Dataset Size", f"{len(df_display)} × {len(df_display.columns)}")
+    if not df_display.empty:
+        data_coverage = (df_display.notna().sum().sum() / (len(df_display) * len(df_display.columns))) * 100
+        st.sidebar.metric("Data Coverage", f"{data_coverage:.1f}%")
+    
     # Main content based on analysis mode
     if analysis_mode == "📊 Overview Dashboard":
         show_overview_dashboard(df_display, stderr_df)
@@ -1347,7 +1473,13 @@ def show_model_performance(df):
     # Model search
     search_term = st.text_input("🔍 Search for models", placeholder="Enter model name or part of name")
     
-    if search_term:
+    # Performance optimization: limit initial display for very large datasets
+    if not search_term and len(df) > 100:
+        st.info(f"📊 **Large dataset detected** ({len(df)} models). Showing top 100 models by average performance. Use search to find specific models.")
+        # Get top 100 models by average score across benchmarks
+        avg_scores = df.mean(axis=1, skipna=True).sort_values(ascending=False)
+        df_display = df.loc[avg_scores.head(100).index]
+    elif search_term:
         matching_models = df.index[df.index.str.contains(search_term, case=False, na=False)]
         if len(matching_models) > 0:
             df_display = df.loc[matching_models]
@@ -1393,12 +1525,20 @@ def show_model_performance(df):
         else:
             min_corr = 0.3
     
-    # Generate rankings
-    ranking_df, rank_matrix, metadata = create_consensus_ranking(
-        df_display, 
-        method=rank_method, 
-        use_rank_imputation=use_rank_imputation
-    )
+    # Generate rankings with progress indicator
+    if use_rank_imputation and len(df_display) > 50:
+        with st.spinner(f"Computing consensus rankings for {len(df_display)} models..."):
+            ranking_df, rank_matrix, metadata = create_consensus_ranking(
+                df_display, 
+                method=rank_method, 
+                use_rank_imputation=use_rank_imputation
+            )
+    else:
+        ranking_df, rank_matrix, metadata = create_consensus_ranking(
+            df_display, 
+            method=rank_method, 
+            use_rank_imputation=use_rank_imputation
+        )
     
     # Display ranking information
     col1, col2 = st.columns(2)
@@ -1451,6 +1591,11 @@ def show_model_performance(df):
             4. Weights based on rank correlation strength (min threshold: {min_corr})
             5. Final consensus rank = median rank across all benchmarks
             
+            **Optimizations**:
+            - Pre-compute correlation matrices for efficiency
+            - Limit to top 5 most correlated benchmarks per prediction
+            - Cache results to avoid recomputation
+            
             **Upsides**:
             - Eliminates bias from models tested only on easier/harder benchmarks
             - Uses the correlation structure to make informed predictions
@@ -1548,78 +1693,15 @@ def show_model_performance(df):
             st.warning(f"Too many models selected ({len(selected_models)}). Please select 10 or fewer models for the radar chart.")
             st.info("💡 **Tip**: Use the search box above to filter models, then select a smaller subset for comparison.")
         else:
-            # Show radar chart for 1-10 models
-            fig = go.Figure()
-            
-            # Use only selected benchmarks
-            clean_benchmark_names = [clean_benchmark_name(b) for b in selected_benchmarks_for_radar]
-            
-            # Define colors for different models
-            colors_list = ['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd', 
-                          '#8c564b', '#e377c2', '#7f7f7f', '#bcbd22', '#17becf']
-            
-            for i, model in enumerate(selected_models):
-                # Get model data for selected benchmarks only
-                model_scores = []
-                for benchmark in selected_benchmarks_for_radar:
-                    score = df_display.loc[model, benchmark]
-                    # Convert to float, use 0.0 for any remaining NaN values
-                    model_scores.append(0.0 if pd.isna(score) else float(score))
-                
-                # Close the radar chart by adding the first value at the end
-                radar_values = model_scores + [model_scores[0]]
-                radar_benchmarks = clean_benchmark_names + [clean_benchmark_names[0]]
-                
-                # Create model name for legend (remove path prefix if present)
-                model_display_name = model.split('/')[-1] if '/' in model else model
-                
-                # Use color from list, cycling if needed
-                model_color = colors_list[i % len(colors_list)]
-                
-                fig.add_trace(go.Scatterpolar(
-                    r=radar_values,
-                    theta=radar_benchmarks,
-                    fill='toself',
-                    name=model_display_name,
-                    line_color=model_color,
-                    hovertemplate='<b>%{theta}</b><br>Score: %{r:.3f}<extra></extra>'
-                ))
-            
-            # Calculate dynamic range for better visualization
-            all_values = []
-            for model in selected_models:
-                for benchmark in selected_benchmarks_for_radar:
-                    score = df_display.loc[model, benchmark]
-                    if not pd.isna(score):
-                        all_values.append(score)
-            
-            if all_values:
-                min_val = min(all_values)
-                max_val = max(all_values)
-                # Add some padding
-                range_padding = (max_val - min_val) * 0.1
-                radar_min = max(0, min_val - range_padding)
-                radar_max = min(1, max_val + range_padding)
+            # Show radar chart for 1-10 models with optimization
+            if len(selected_models) > 3 or len(selected_benchmarks_for_radar) > 8:
+                with st.spinner("Generating radar chart..."):
+                    fig = create_optimized_radar_chart(df_display, selected_models, selected_benchmarks_for_radar)
             else:
-                radar_min, radar_max = 0, 1
-            
-            # Adjust chart size based on number of models
-            chart_height = 600 if len(selected_models) <= 3 else 700
-            
-            fig.update_layout(
-                polar=dict(
-                    radialaxis=dict(
-                        visible=True,
-                        range=[radar_min, radar_max],
-                        tickformat='.2f'
-                    )),
-                showlegend=True,
-                title=f"Model Performance Radar Chart ({len(selected_benchmarks_for_radar)} benchmarks, {len(selected_models)} models)",
-                width=700,
-                height=chart_height
-            )
+                fig = create_optimized_radar_chart(df_display, selected_models, selected_benchmarks_for_radar)
             
-            st.plotly_chart(fig, use_container_width=True)
+            if fig:
+                st.plotly_chart(fig, use_container_width=True)
             
             # Add explanation about missing values (only if not using complete data only)
             if not complete_data_only: