Upload 12 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+global_power_plant_database.csv filter=lfs diff=lfs merge=lfs -text

analyzing.py

@@ -0,0 +1,675 @@
+import streamlit as st
+import cudf.pandas
+cudf.pandas.install()
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import seaborn as sns
+from uap_analyzer import UAPParser, UAPAnalyzer, UAPVisualizer
+# import ChartGen
+# from ChartGen import ChartGPT
+from Levenshtein import distance
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import confusion_matrix
+from stqdm import stqdm
+stqdm.pandas()
+import streamlit.components.v1 as components
+from dateutil import parser
+from sentence_transformers import SentenceTransformer
+import torch
+import squarify
+import matplotlib.colors as mcolors
+import textwrap
+import datamapplot
+
+st.set_option('deprecation.showPyplotGlobalUse', False)
+
+from pandas.api.types import (
+    is_categorical_dtype,
+    is_datetime64_any_dtype,
+    is_numeric_dtype,
+    is_object_dtype,
+)
+
+
+
+def load_data(file_path, key='df'):
+    return pd.read_hdf(file_path, key=key)
+
+
+def gemini_query(question, selected_data, gemini_key):
+
+    if question == "":
+        question = "Summarize the following data in relevant bullet points"
+
+    import pathlib
+    import textwrap
+
+    import google.generativeai as genai
+
+    from IPython.display import display
+    from IPython.display import Markdown
+
+
+    def to_markdown(text):
+        text = text.replace('•', '  *')
+        return Markdown(textwrap.indent(text, '> ', predicate=lambda _: True))
+    
+    # selected_data is a list
+    # remove empty
+
+    filtered = [str(x) for x in selected_data if str(x) != '' and x is not None]
+    # make a string
+    context = '\n'.join(filtered)
+
+    genai.configure(api_key=gemini_key)
+    query_model = genai.GenerativeModel('models/gemini-1.5-pro-latest')
+    response = query_model.generate_content([f"{question}\n Answer based on this context: {context}\n\n"])
+    return(response.text)
+
+def plot_treemap(df, column, top_n=32):
+        # Get the value counts and the top N labels
+        value_counts = df[column].value_counts()
+        top_labels = value_counts.iloc[:top_n].index
+        
+        # Use np.where to replace all values not in the top N with 'Other'
+        revised_column = f'{column}_revised'
+        df[revised_column] = np.where(df[column].isin(top_labels), df[column], 'Other')
+
+        # Get the value counts including the 'Other' category
+        sizes = df[revised_column].value_counts().values
+        labels = df[revised_column].value_counts().index
+
+        # Get a gradient of colors
+        # colors = list(mcolors.TABLEAU_COLORS.values())
+
+        n_colors = len(sizes)
+        colors = plt.cm.Oranges(np.linspace(0.3, 0.9, n_colors))[::-1]
+
+
+        # Get % of each category
+        percents = sizes / sizes.sum()
+
+        # Prepare labels with percentages
+        labels = [f'{label}\n {percent:.1%}' for label, percent in zip(labels, percents)]
+
+        fig, ax = plt.subplots(figsize=(20, 12))
+
+        # Plot the treemap
+        squarify.plot(sizes=sizes, label=labels, alpha=0.7, pad=True, color=colors, text_kwargs={'fontsize': 10})
+
+        ax = plt.gca()
+        # Iterate over text elements and rectangles (patches) in the axes for color adjustment
+        for text, rect in zip(ax.texts, ax.patches):
+            background_color = rect.get_facecolor()
+            r, g, b, _ = mcolors.to_rgba(background_color)
+            brightness = np.average([r, g, b])
+            text.set_color('white' if brightness < 0.5 else 'black')
+
+            # Adjust font size based on rectangle's area and wrap long text
+            coef = 0.8
+            font_size = np.sqrt(rect.get_width() * rect.get_height()) * coef
+            text.set_fontsize(font_size)
+            wrapped_text = textwrap.fill(text.get_text(), width=20)
+            text.set_text(wrapped_text)
+
+        plt.axis('off')
+        plt.gca().invert_yaxis()
+        plt.gcf().set_size_inches(20, 12)
+
+        fig.patch.set_alpha(0)
+
+        ax.patch.set_alpha(0)
+        return fig
+
+def plot_hist(df, column, bins=10, kde=True):
+        fig, ax = plt.subplots(figsize=(12, 6))
+        sns.histplot(data=df, x=column, kde=True, bins=bins,color='orange')
+        # set the ticks and frame in orange
+        ax.spines['bottom'].set_color('orange')
+        ax.spines['top'].set_color('orange')
+        ax.spines['right'].set_color('orange')
+        ax.spines['left'].set_color('orange')
+        ax.xaxis.label.set_color('orange')
+        ax.yaxis.label.set_color('orange')
+        ax.tick_params(axis='x', colors='orange')
+        ax.tick_params(axis='y', colors='orange')
+        ax.title.set_color('orange')
+
+        # Set transparent background
+        fig.patch.set_alpha(0)
+        ax.patch.set_alpha(0)
+        return fig
+
+
+
+
+def plot_line(df, x_column, y_columns, figsize=(12, 10), color='orange', title=None, rolling_mean_value=2):
+    import matplotlib.cm as cm
+    # Sort the dataframe by the date column
+    df = df.sort_values(by=x_column)
+
+    # Calculate rolling mean for each y_column
+    if rolling_mean_value:
+        df[y_columns] = df[y_columns].rolling(len(df) // rolling_mean_value).mean()
+
+    # Create the plot
+    fig, ax = plt.subplots(figsize=figsize)
+
+    colors = cm.Oranges(np.linspace(0.2, 1, len(y_columns)))
+
+    # Plot each y_column as a separate line with a different color
+    for i, y_column in enumerate(y_columns):
+        df.plot(x=x_column, y=y_column, ax=ax, color=colors[i], label=y_column, linewidth=.5)
+
+    # Rotate x-axis labels
+    ax.set_xticklabels(ax.get_xticklabels(), rotation=30, ha='right')
+
+    # Format x_column as date if it is
+    if np.issubdtype(df[x_column].dtype, np.datetime64) or np.issubdtype(df[x_column].dtype, np.timedelta64):
+        df[x_column] = pd.to_datetime(df[x_column]).dt.date
+
+    # Set title, labels, and legend
+    ax.set_title(title or f'{", ".join(y_columns)} over {x_column}', color=color, fontweight='bold')
+    ax.set_xlabel(x_column, color=color)
+    ax.set_ylabel(', '.join(y_columns), color=color)
+    ax.spines['bottom'].set_color('orange')
+    ax.spines['top'].set_color('orange')
+    ax.spines['right'].set_color('orange')
+    ax.spines['left'].set_color('orange')
+    ax.xaxis.label.set_color('orange')
+    ax.yaxis.label.set_color('orange')
+    ax.tick_params(axis='x', colors='orange')
+    ax.tick_params(axis='y', colors='orange')
+    ax.title.set_color('orange')
+
+    ax.legend(loc='upper right', bbox_to_anchor=(1, 1), facecolor='black', framealpha=.4, labelcolor='orange', edgecolor='orange')
+
+    # Remove background
+    fig.patch.set_alpha(0)
+    ax.patch.set_alpha(0)
+
+    return fig
+
+def plot_bar(df, x_column, y_column, figsize=(12, 10), color='orange', title=None):
+    fig, ax = plt.subplots(figsize=figsize)
+
+    sns.barplot(data=df, x=x_column, y=y_column, color=color, ax=ax)
+
+    ax.set_title(title if title else f'{y_column} by {x_column}', color=color, fontweight='bold')
+    ax.set_xlabel(x_column, color=color)
+    ax.set_ylabel(y_column, color=color)
+
+    ax.tick_params(axis='x', colors=color)
+    ax.tick_params(axis='y', colors=color)
+
+    # Remove background
+    fig.patch.set_alpha(0)
+    ax.patch.set_alpha(0)
+    ax.spines['bottom'].set_color('orange')
+    ax.spines['top'].set_color('orange')
+    ax.spines['right'].set_color('orange')
+    ax.spines['left'].set_color('orange')
+    ax.xaxis.label.set_color('orange')
+    ax.yaxis.label.set_color('orange')
+    ax.tick_params(axis='x', colors='orange')
+    ax.tick_params(axis='y', colors='orange')
+    ax.title.set_color('orange')
+    ax.legend(loc='upper right', bbox_to_anchor=(1, 1), facecolor='black', framealpha=.4, labelcolor='orange', edgecolor='orange')
+
+    return fig
+
+def plot_grouped_bar(df, x_columns, y_column, figsize=(12, 10), colors=None, title=None):
+    fig, ax = plt.subplots(figsize=figsize)
+
+    width = 0.8 / len(x_columns)  # the width of the bars
+    x = np.arange(len(df))  # the label locations
+
+    for i, x_column in enumerate(x_columns):
+        sns.barplot(data=df, x=x, y=y_column, color=colors[i] if colors else None, ax=ax, width=width, label=x_column)
+        x += width  # add the width of the bar to the x position for the next bar
+
+    ax.set_title(title if title else f'{y_column} by {", ".join(x_columns)}', color='orange', fontweight='bold')
+    ax.set_xlabel('Groups', color='orange')
+    ax.set_ylabel(y_column, color='orange')
+
+    ax.set_xticks(x - width * len(x_columns) / 2)
+    ax.set_xticklabels(df.index)
+
+    ax.tick_params(axis='x', colors='orange')
+    ax.tick_params(axis='y', colors='orange')
+
+    # Remove background
+    fig.patch.set_alpha(0)
+    ax.patch.set_alpha(0)
+    ax.spines['bottom'].set_color('orange')
+    ax.spines['top'].set_color('orange')
+    ax.spines['right'].set_color('orange')
+    ax.spines['left'].set_color('orange')
+    ax.xaxis.label.set_color('orange')
+    ax.yaxis.label.set_color('orange')
+    ax.title.set_color('orange')
+    ax.legend(loc='upper right', bbox_to_anchor=(1, 1), facecolor='black', framealpha=.4, labelcolor='orange', edgecolor='orange')
+
+    return fig
+
+
+def filter_dataframe(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Adds a UI on top of a dataframe to let viewers filter columns
+
+    Args:
+        df (pd.DataFrame): Original dataframe
+
+    Returns:
+        pd.DataFrame: Filtered dataframe
+    """
+
+    title_font = "Arial"
+    body_font = "Arial"
+    title_size = 32
+    colors = ["red", "green", "blue"]
+    interpretation = False
+    extract_docx = False
+    title = "My Chart"
+    regex = ".*"
+    img_path = 'default_image.png'
+
+
+    #try:
+    #    modify = st.checkbox("Add filters on raw data")
+    #except:
+    #    try:
+    #        modify = st.checkbox("Add filters on processed data")
+    #    except:
+    #        try:
+    #            modify = st.checkbox("Add filters on parsed data")
+    #        except:
+    #            pass
+
+    #if not modify:
+    #    return df
+
+    df_ = df.copy()
+    # Try to convert datetimes into a standard format (datetime, no timezone)
+
+#modification_container = st.container()
+
+#with modification_container:
+    try:
+        to_filter_columns = st.multiselect("Filter dataframe on", df_.columns)
+    except:
+        try:
+            to_filter_columns = st.multiselect("Filter dataframe", df_.columns)
+        except:
+            try:
+                to_filter_columns = st.multiselect("Filter the dataframe on", df_.columns)
+            except:
+                pass
+
+    date_column = None
+    filtered_columns = []
+
+    for column in to_filter_columns:
+        left, right = st.columns((1, 20))
+        # Treat columns with < 200 unique values as categorical if not date or numeric
+        if is_categorical_dtype(df_[column]) or (df_[column].nunique() < 120 and not is_datetime64_any_dtype(df_[column]) and not is_numeric_dtype(df_[column])):
+            user_cat_input = right.multiselect(
+                f"Values for {column}",
+                df_[column].value_counts().index.tolist(),
+                default=list(df_[column].value_counts().index)
+            )
+            df_ = df_[df_[column].isin(user_cat_input)]
+            filtered_columns.append(column)
+
+            with st.status(f"Category Distribution: {column}", expanded=False) as stat:
+                st.pyplot(plot_treemap(df_, column))
+
+        elif is_numeric_dtype(df_[column]):
+            _min = float(df_[column].min())
+            _max = float(df_[column].max())
+            step = (_max - _min) / 100
+            user_num_input = right.slider(
+                f"Values for {column}",
+                min_value=_min,
+                max_value=_max,
+                value=(_min, _max),
+                step=step,
+            )
+            df_ = df_[df_[column].between(*user_num_input)]
+            filtered_columns.append(column)
+
+            # Chart_GPT = ChartGPT(df_, title_font, body_font, title_size,
+            #      colors, interpretation, extract_docx, img_path)
+
+            with st.status(f"Numerical Distribution: {column}", expanded=False) as stat_:
+                st.pyplot(plot_hist(df_, column, bins=int(round(len(df_[column].unique())-1)/2)))
+
+        elif is_object_dtype(df_[column]):
+            try:
+                df_[column] = pd.to_datetime(df_[column], infer_datetime_format=True, errors='coerce')
+            except Exception:
+                try:
+                    df_[column] = df_[column].apply(parser.parse)
+                except Exception:
+                    pass
+
+            if is_datetime64_any_dtype(df_[column]):
+                df_[column] = df_[column].dt.tz_localize(None)
+                min_date = df_[column].min().date()
+                max_date = df_[column].max().date()
+                user_date_input = right.date_input(
+                    f"Values for {column}",
+                    value=(min_date, max_date),
+                    min_value=min_date,
+                    max_value=max_date,
+                )
+                # if len(user_date_input) == 2:
+                #     start_date, end_date = user_date_input
+                #     df_ = df_.loc[df_[column].dt.date.between(start_date, end_date)]
+                if len(user_date_input) == 2:
+                    user_date_input = tuple(map(pd.to_datetime, user_date_input))
+                    start_date, end_date = user_date_input
+                    df_ = df_.loc[df_[column].between(start_date, end_date)]
+
+                date_column = column
+
+                if date_column and filtered_columns:
+                    numeric_columns = [col for col in filtered_columns if is_numeric_dtype(df_[col])]
+                    if numeric_columns:
+                        fig = plot_line(df_, date_column, numeric_columns)
+                        #st.pyplot(fig)
+                    # now to deal with categorical columns
+                    categorical_columns = [col for col in filtered_columns if is_categorical_dtype(df_[col])]
+                    if categorical_columns:
+                        fig2 = plot_bar(df_, date_column, categorical_columns[0])
+                        #st.pyplot(fig2)
+                    with st.status(f"Date Distribution: {column}", expanded=False) as stat:
+                        try:
+                            st.pyplot(fig)
+                        except Exception as e:
+                            st.error(f"Error plotting line chart: {e}")
+                            pass
+                        try:
+                            st.pyplot(fig2)
+                        except Exception as e:
+                            st.error(f"Error plotting bar chart: {e}")
+
+
+        else:
+            user_text_input = right.text_input(
+                f"Substring or regex in {column}",
+            )
+            if user_text_input:
+                df_ = df_[df_[column].astype(str).str.contains(user_text_input)]
+    # write len of df after filtering with % of original
+    st.write(f"{len(df_)} rows ({len(df_) / len(df) * 100:.2f}%)")
+    return df_
+
+def merge_clusters(df, column):
+    cluster_terms_ = df.__dict__['cluster_terms']
+    cluster_labels_ = df.__dict__['cluster_labels']
+    label_name_map = {label: cluster_terms_[label] for label in set(cluster_labels_)}
+    merge_map = {}
+    # Iterate over term pairs and decide on merging based on the distance
+    for idx, term1 in enumerate(cluster_terms_):
+        for jdx, term2 in enumerate(cluster_terms_):
+            if idx < jdx and distance(term1, term2) <= 3:  # Adjust threshold as needed
+                # Decide to merge labels corresponding to jdx into labels corresponding to idx
+                # Find labels corresponding to jdx and idx
+                labels_to_merge = [label for label, term_index in enumerate(cluster_labels_) if term_index == jdx]
+                for label in labels_to_merge:
+                    merge_map[label] = idx  # Map the label to use the term index of term1
+
+    # Update the analyzer with the merged numeric labels 
+    updated_cluster_labels_ = [merge_map[label] if label in merge_map else label for label in cluster_labels_]
+
+    df.__dict__['cluster_labels'] = updated_cluster_labels_
+    # Optional: Update string labels to reflect merged labels
+    updated_string_labels = [cluster_terms_[label] for label in updated_cluster_labels_]
+    df.__dict__['string_labels'] = updated_string_labels
+    return updated_string_labels
+
+def analyze_and_predict(data, analyzers, col_names, clusters):
+    visualizer = UAPVisualizer()
+    new_data = pd.DataFrame()
+    for i, column  in enumerate(col_names):
+        #new_data[f'Analyzer_{column}'] = analyzers[column].__dict__['cluster_labels']
+        new_data[f'Analyzer_{column}'] = clusters[column]
+        data[f'Analyzer_{column}'] = clusters[column]
+        #data[f'Analyzer_{column}'] = analyzer.__dict__['cluster_labels']
+
+        print(f"Cluster terms extracted for {column}")
+
+    for col in data.columns:
+        if 'Analyzer' in col:
+            data[col] = data[col].astype('category')
+
+    new_data = new_data.fillna('null').astype('category')
+    data_nums = new_data.apply(lambda x: x.cat.codes)
+
+    for col in data_nums.columns:
+        try:
+            categories = new_data[col].cat.categories
+            x_train, x_test, y_train, y_test = train_test_split(data_nums.drop(columns=[col]), data_nums[col], test_size=0.2, random_state=42)
+            bst, accuracy, preds = visualizer.train_xgboost(x_train, y_train, x_test, y_test, len(categories))
+            fig = visualizer.plot_results(new_data, bst, x_test, y_test, preds, categories, accuracy, col)
+            with st.status(f"Charts Analyses: {col}", expanded=True) as status:
+                st.pyplot(fig)
+                status.update(label=f"Chart Processed: {col}", expanded=False)   
+        except Exception as e:
+            print(f"Error processing {col}: {e}")
+            continue
+    return new_data, data
+
+from config import API_KEY, GEMINI_KEY, FORMAT_LONG
+
+with torch.no_grad():
+    torch.cuda.empty_cache()
+
+#st.set_page_config(
+#    page_title="UAP ANALYSIS",
+#    page_icon=":alien:",
+#    layout="wide",
+#    initial_sidebar_state="expanded",
+#)
+
+st.title('UAP Analysis Dashboard')
+
+# Initialize session state
+if 'analyzers' not in st.session_state:
+    st.session_state['analyzers'] = []
+if 'col_names' not in st.session_state:
+    st.session_state['col_names'] = []
+if 'clusters' not in st.session_state:
+    st.session_state['clusters'] = {}
+if 'new_data' not in st.session_state:
+    st.session_state['new_data'] = pd.DataFrame()
+if 'dataset' not in st.session_state:
+    st.session_state['dataset'] = pd.DataFrame()
+if 'data_processed' not in st.session_state:
+    st.session_state['data_processed'] = False
+if 'stage' not in st.session_state:
+    st.session_state['stage'] = 0
+if 'filtered_data' not in st.session_state:
+    st.session_state['filtered_data'] = None
+if 'gemini_answer' not in st.session_state:
+    st.session_state['gemini_answer'] = None
+if 'parsed_responses' not in st.session_state:
+    st.session_state['parsed_responses'] = None
+
+# Load dataset
+data_path = 'uap_files_embeds.h5'
+
+my_dataset = st.file_uploader("Upload Parsed DataFrame", type=["csv", "xlsx"])
+# if st.session_state['parsed_responses'] is not None:
+#     #use_parsed_data = st.checkbox('Analyze recently parsed dataset')
+#     #if use_parsed_data:
+#     parsed = st.session_state.get('parsed_responses', pd.DataFrame()).copy() # this will overwrite the parsed_responses variable
+#     filtered_data = filter_dataframe(parsed)
+#     st.dataframe(filtered_data)
+if my_dataset is not None:
+#     try:
+#         data = pd.read_csv(my_dataset) if my_dataset.type == "text/csv" else pd.read_excel(my_dataset)
+#         filtered_data = filter_dataframe(data)
+#         st.dataframe(filtered_data)
+#     except Exception as e:
+#         st.error(f"An error occurred while reading the file: {e}")
+# #if 'parsed_responses' not in st.session_state:
+    try:
+        if my_dataset.type == "text/csv":
+            data = pd.read_csv(my_dataset)
+        elif my_dataset.type == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet":
+            data = pd.read_excel(my_dataset)
+        else:
+            st.error("Unsupported file type. Please upload a CSV, Excel or HD5 file.")
+            st.stop()
+        parser = filter_dataframe(data)
+        st.session_state['parsed_responses'] = parser
+        st.dataframe(parser)
+        st.success(f"Successfully loaded and displayed data from {my_dataset.name}")
+    except Exception as e:
+        st.error(f"An error occurred while reading the file: {e}")
+else:
+    parsed = load_data(data_path).drop(columns=['embeddings']).head(10000)
+    parsed_responses = filter_dataframe(parsed)
+    st.session_state['parsed_responses'] = parsed_responses
+    st.dataframe(parsed_responses)
+col1, col2 = st.columns(2)
+with col1:
+    col_parsed = st.selectbox("Which column do you want to query?", st.session_state['parsed_responses'].columns)
+with col2:
+    GEMINI_KEY = st.text_input('Gemini API Key', GEMINI_KEY, type='password', help="Enter your Gemini API key")
+
+if col_parsed and GEMINI_KEY:
+    selected_column_data = st.session_state['parsed_responses'][col_parsed].tolist()
+    question = st.text_input("Ask a question or leave empty for summarization")
+    if st.button("Generate Query") and selected_column_data:
+        st.write(gemini_query(question, selected_column_data, GEMINI_KEY))
+st.session_state['stage'] = 1
+
+
+if st.session_state['stage'] > 0 :
+    columns_to_analyze = st.multiselect(
+        label='Select columns to analyze',
+        options=st.session_state['parsed_responses'].columns
+    )
+    if columns_to_analyze:
+        analyzers = []
+        col_names = []
+        clusters = {}
+        for column in columns_to_analyze:
+            with torch.no_grad():    
+                with st.status(f"Processing {column}", expanded=True) as status:
+                    analyzer = UAPAnalyzer(st.session_state['parsed_responses'], column)
+                    st.write(f"Processing {column}...")
+                    analyzer.preprocess_data(top_n=32)
+                    st.write("Reducing dimensionality...")
+                    analyzer.reduce_dimensionality(method='UMAP', n_components=2, n_neighbors=15, min_dist=0.1)
+                    st.write("Clustering data...")
+                    analyzer.cluster_data(method='HDBSCAN', min_cluster_size=15)
+                    analyzer.get_tf_idf_clusters(top_n=3)
+                    st.write("Naming clusters...")
+                    analyzers.append(analyzer)
+                    col_names.append(column)
+                    clusters[column] = analyzer.merge_similar_clusters(cluster_terms=analyzer.__dict__['cluster_terms'], cluster_labels=analyzer.__dict__['cluster_labels'])
+                    
+                    # Run the visualization
+                    # fig = datamapplot.create_plot(
+                    #     analyzer.__dict__['reduced_embeddings'],
+                    #     analyzer.__dict__['cluster_labels'].astype(str),
+                    #     #label_font_size=11,
+                    #     label_wrap_width=20,
+                    #     use_medoids=True,
+                    # )#.to_html(full_html=False, include_plotlyjs='cdn')
+                    # st.pyplot(fig.savefig())
+                    status.update(label=f"Processing {column} complete", expanded=False)
+        st.session_state['analyzers'] = analyzers
+        st.session_state['col_names'] = col_names
+        st.session_state['clusters'] = clusters
+        
+        # save space
+        parsed = None
+        analyzers = None
+        col_names = None
+        clusters = None
+
+        if st.session_state['clusters'] is not None:
+            try:
+                new_data, parsed_responses = analyze_and_predict(st.session_state['parsed_responses'], st.session_state['analyzers'], st.session_state['col_names'], st.session_state['clusters'])       
+                st.session_state['dataset'] = parsed_responses
+                st.session_state['new_data'] = new_data
+                st.session_state['data_processed'] = True
+            except Exception as e:
+                st.write(f"Error processing data: {e}")
+    
+        if st.session_state['data_processed']:
+            try:
+                visualizer = UAPVisualizer(data=st.session_state['new_data'])
+                #new_data = pd.DataFrame()  # Assuming new_data is prepared earlier in the code
+                fig2 = visualizer.plot_cramers_v_heatmap(data=st.session_state['new_data'], significance_level=0.05)
+                with st.status(f"Cramer's V Chart", expanded=True) as statuss:
+                    st.pyplot(fig2)
+                    statuss.update(label="Cramer's V chart plotted", expanded=False)   
+            except Exception as e:
+                st.write(f"Error plotting Cramers V: {e}")
+
+            for i, column in enumerate(st.session_state['col_names']):
+                #if stateful_button(f"Show {column} clusters {i}", key=f"show_{column}_clusters"):
+                # if st.session_state['data_processed']:
+                #     with st.status(f"Show clusters {column}", expanded=True) as stats:
+                #         fig3 = st.session_state['analyzers'][i].plot_embeddings4(title=f"{column} clusters", cluster_terms=st.session_state['analyzers'][i].__dict__['cluster_terms'], cluster_labels=st.session_state['analyzers'][i].__dict__['cluster_labels'], reduced_embeddings=st.session_state['analyzers'][i].__dict__['reduced_embeddings'], column=f'Analyzer_{column}', data=st.session_state['new_data'])
+                #         stats.update(label=f"Show clusters {column} complete", expanded=False)
+                if st.session_state['data_processed']:
+                    with st.status(f"Show clusters {column}", expanded=True) as stats:
+                        fig3 = st.session_state['analyzers'][i].plot_embeddings4(
+                            title=f"{column} clusters", 
+                            cluster_terms=st.session_state['analyzers'][i].__dict__['cluster_terms'], 
+                            cluster_labels=st.session_state['analyzers'][i].__dict__['cluster_labels'], 
+                            reduced_embeddings=st.session_state['analyzers'][i].__dict__['reduced_embeddings'], 
+                            column=column,  # Use the original column name here
+                            data=st.session_state['parsed_responses']  # Use the original dataset here
+                        )
+                        stats.update(label=f"Show clusters {column} complete", expanded=False)
+                st.session_state['analysis_complete'] = True
+
+
+# this will check if the dataframe is not empty
+# if st.session_state['new_data'] is not None:
+#     parsed2 = st.session_state.get('dataset', pd.DataFrame())
+#     parsed2 = filter_dataframe(parsed2)
+#     col1, col2 = st.columns(2)
+#     st.dataframe(parsed2)
+#     with col1:
+#         col_parsed2 = st.selectbox("Which columns do you want to query?", parsed2.columns)
+#     with col2:
+#         GEMINI_KEY = st.text_input('Gemini APIs Key', GEMINI_KEY, type='password', help="Enter your Gemini API key")
+#     if col_parsed and GEMINI_KEY:
+#         selected_column_data2 = parsed2[col_parsed2].tolist()
+#         question2 = st.text_input("Ask a questions or leave empty for summarization")
+#         if st.button("Generate Query") and selected_column_data2:
+#             with st.status(f"Generating Query", expanded=True) as status:
+#                 gemini_answer = gemini_query(question2, selected_column_data2, GEMINI_KEY)
+#                 st.write(gemini_answer)
+#                 st.session_state['gemini_answer'] = gemini_answer
+
+if 'analysis_complete' in st.session_state and st.session_state['analysis_complete']:
+    ticked_analysis = st.checkbox('Query Processed Data')
+    if ticked_analysis:
+        if st.session_state['new_data'] is not None:
+            parsed2 = st.session_state.get('dataset', pd.DataFrame()).copy()
+            parsed2 = filter_dataframe(parsed2)
+            col1, col2 = st.columns(2)
+            st.dataframe(parsed2)
+            with col1:
+                col_parsed2 = st.selectbox("Which columns do you want to query?", parsed2.columns)
+            with col2:
+                GEMINI_KEY = st.text_input('Gemini APIs Key', GEMINI_KEY, type='password', help="Enter your Gemini API key")
+            if col_parsed2 and GEMINI_KEY:
+                selected_column_data2 = parsed2[col_parsed2].tolist()
+                question2 = st.text_input("Ask a questions or leave empty for summarization")
+                if st.button("Generate Queries") and selected_column_data2:
+                    with st.status(f"Generating Query", expanded=True) as status:
+                        gemini_answer = gemini_query(question2, selected_column_data2, GEMINI_KEY)
+                        st.write(gemini_answer)
+                        st.session_state['gemini_answer'] = gemini_answer

final_ufoseti_dataset.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:829fb6660b24626eb5db39952783c6e17dc17c7c4636df0dfc8b641d0c84efe5
+size 39219544

global_power_plant_database.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee79da6a4e0948e0df5ffc9fd372ee453e7e1d0b2ead57f568565750014f7d59
+size 12758630

magnetic.py

@@ -0,0 +1,907 @@
+
+import math
+import pandas as pd
+import numpy as np
+import json
+import requests
+import datetime
+from datetime import timedelta
+from PIL import Image
+# alternative to PIL
+import matplotlib.pyplot as plt
+import matplotlib.image as mpimg
+import os
+import matplotlib.dates as mdates
+import seaborn as sns
+from IPython.display import Image as image_display
+path = os.getcwd()
+from fastdtw import fastdtw
+from scipy.spatial.distance import euclidean
+from IPython.display import display
+from dateutil import parser
+from Levenshtein import distance
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import confusion_matrix
+from stqdm import stqdm
+stqdm.pandas()
+import streamlit.components.v1 as components
+from dateutil import parser
+from sentence_transformers import SentenceTransformer
+import torch
+import squarify
+import matplotlib.colors as mcolors
+import textwrap
+import datamapplot
+import streamlit as st
+
+
+st.title('Magnetic Correlations Dashboard')
+
+st.set_option('deprecation.showPyplotGlobalUse', False)
+
+
+from pandas.api.types import (
+    is_categorical_dtype,
+    is_datetime64_any_dtype,
+    is_numeric_dtype,
+    is_object_dtype,
+)
+
+
+def plot_treemap(df, column, top_n=32):
+        # Get the value counts and the top N labels
+        value_counts = df[column].value_counts()
+        top_labels = value_counts.iloc[:top_n].index
+        
+        # Use np.where to replace all values not in the top N with 'Other'
+        revised_column = f'{column}_revised'
+        df[revised_column] = np.where(df[column].isin(top_labels), df[column], 'Other')
+
+        # Get the value counts including the 'Other' category
+        sizes = df[revised_column].value_counts().values
+        labels = df[revised_column].value_counts().index
+
+        # Get a gradient of colors
+        # colors = list(mcolors.TABLEAU_COLORS.values())
+
+        n_colors = len(sizes)
+        colors = plt.cm.Oranges(np.linspace(0.3, 0.9, n_colors))[::-1]
+
+
+        # Get % of each category
+        percents = sizes / sizes.sum()
+
+        # Prepare labels with percentages
+        labels = [f'{label}\n {percent:.1%}' for label, percent in zip(labels, percents)]
+
+        fig, ax = plt.subplots(figsize=(20, 12))
+
+        # Plot the treemap
+        squarify.plot(sizes=sizes, label=labels, alpha=0.7, pad=True, color=colors, text_kwargs={'fontsize': 10})
+
+        ax = plt.gca()
+        # Iterate over text elements and rectangles (patches) in the axes for color adjustment
+        for text, rect in zip(ax.texts, ax.patches):
+            background_color = rect.get_facecolor()
+            r, g, b, _ = mcolors.to_rgba(background_color)
+            brightness = np.average([r, g, b])
+            text.set_color('white' if brightness < 0.5 else 'black')
+
+
+def plot_hist(df, column, bins=10, kde=True):
+        fig, ax = plt.subplots(figsize=(12, 6))
+        sns.histplot(data=df, x=column, kde=True, bins=bins,color='orange')
+        # set the ticks and frame in orange
+        ax.spines['bottom'].set_color('orange')
+        ax.spines['top'].set_color('orange')
+        ax.spines['right'].set_color('orange')
+        ax.spines['left'].set_color('orange')
+        ax.xaxis.label.set_color('orange')
+        ax.yaxis.label.set_color('orange')
+        ax.tick_params(axis='x', colors='orange')
+        ax.tick_params(axis='y', colors='orange')
+        ax.title.set_color('orange')
+
+        # Set transparent background
+        fig.patch.set_alpha(0)
+        ax.patch.set_alpha(0)
+        return fig
+
+
+
+
+def plot_line(df, x_column, y_columns, figsize=(12, 10), color='orange', title=None, rolling_mean_value=2):
+    import matplotlib.cm as cm
+    # Sort the dataframe by the date column
+    df = df.sort_values(by=x_column)
+
+    # Calculate rolling mean for each y_column
+    if rolling_mean_value:
+        df[y_columns] = df[y_columns].rolling(len(df) // rolling_mean_value).mean()
+
+    # Create the plot
+    fig, ax = plt.subplots(figsize=figsize)
+
+    colors = cm.Oranges(np.linspace(0.2, 1, len(y_columns)))
+
+    # Plot each y_column as a separate line with a different color
+    for i, y_column in enumerate(y_columns):
+        df.plot(x=x_column, y=y_column, ax=ax, color=colors[i], label=y_column, linewidth=.5)
+
+    # Rotate x-axis labels
+    ax.set_xticklabels(ax.get_xticklabels(), rotation=30, ha='right')
+
+    # Format x_column as date if it is
+    if np.issubdtype(df[x_column].dtype, np.datetime64) or np.issubdtype(df[x_column].dtype, np.timedelta64):
+        df[x_column] = pd.to_datetime(df[x_column]).dt.date
+
+    # Set title, labels, and legend
+    ax.set_title(title or f'{", ".join(y_columns)} over {x_column}', color=color, fontweight='bold')
+    ax.set_xlabel(x_column, color=color)
+    ax.set_ylabel(', '.join(y_columns), color=color)
+    ax.spines['bottom'].set_color('orange')
+    ax.spines['top'].set_color('orange')
+    ax.spines['right'].set_color('orange')
+    ax.spines['left'].set_color('orange')
+    ax.xaxis.label.set_color('orange')
+    ax.yaxis.label.set_color('orange')
+    ax.tick_params(axis='x', colors='orange')
+    ax.tick_params(axis='y', colors='orange')
+    ax.title.set_color('orange')
+
+    ax.legend(loc='upper right', bbox_to_anchor=(1, 1), facecolor='black', framealpha=.4, labelcolor='orange', edgecolor='orange')
+
+    # Remove background
+    fig.patch.set_alpha(0)
+    ax.patch.set_alpha(0)
+
+    return fig
+
+def plot_bar(df, x_column, y_column, figsize=(12, 10), color='orange', title=None):
+    fig, ax = plt.subplots(figsize=figsize)
+
+    sns.barplot(data=df, x=x_column, y=y_column, color=color, ax=ax)
+
+    ax.set_title(title if title else f'{y_column} by {x_column}', color=color, fontweight='bold')
+    ax.set_xlabel(x_column, color=color)
+    ax.set_ylabel(y_column, color=color)
+
+    ax.tick_params(axis='x', colors=color)
+    ax.tick_params(axis='y', colors=color)
+
+    # Remove background
+    fig.patch.set_alpha(0)
+    ax.patch.set_alpha(0)
+    ax.spines['bottom'].set_color('orange')
+    ax.spines['top'].set_color('orange')
+    ax.spines['right'].set_color('orange')
+    ax.spines['left'].set_color('orange')
+    ax.xaxis.label.set_color('orange')
+    ax.yaxis.label.set_color('orange')
+    ax.tick_params(axis='x', colors='orange')
+    ax.tick_params(axis='y', colors='orange')
+    ax.title.set_color('orange')
+    ax.legend(loc='upper right', bbox_to_anchor=(1, 1), facecolor='black', framealpha=.4, labelcolor='orange', edgecolor='orange')
+
+    return fig
+
+def plot_grouped_bar(df, x_columns, y_column, figsize=(12, 10), colors=None, title=None):
+    fig, ax = plt.subplots(figsize=figsize)
+
+    width = 0.8 / len(x_columns)  # the width of the bars
+    x = np.arange(len(df))  # the label locations
+
+    for i, x_column in enumerate(x_columns):
+        sns.barplot(data=df, x=x, y=y_column, color=colors[i] if colors else None, ax=ax, width=width, label=x_column)
+        x += width  # add the width of the bar to the x position for the next bar
+
+    ax.set_title(title if title else f'{y_column} by {", ".join(x_columns)}', color='orange', fontweight='bold')
+    ax.set_xlabel('Groups', color='orange')
+    ax.set_ylabel(y_column, color='orange')
+
+    ax.set_xticks(x - width * len(x_columns) / 2)
+    ax.set_xticklabels(df.index)
+
+    ax.tick_params(axis='x', colors='orange')
+    ax.tick_params(axis='y', colors='orange')
+
+    # Remove background
+    fig.patch.set_alpha(0)
+    ax.patch.set_alpha(0)
+    ax.spines['bottom'].set_color('orange')
+    ax.spines['top'].set_color('orange')
+    ax.spines['right'].set_color('orange')
+    ax.spines['left'].set_color('orange')
+    ax.xaxis.label.set_color('orange')
+    ax.yaxis.label.set_color('orange')
+    ax.title.set_color('orange')
+    ax.legend(loc='upper right', bbox_to_anchor=(1, 1), facecolor='black', framealpha=.4, labelcolor='orange', edgecolor='orange')
+
+    return fig
+
+
+def filter_dataframe(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Adds a UI on top of a dataframe to let viewers filter columns
+
+    Args:
+        df (pd.DataFrame): Original dataframe
+
+    Returns:
+        pd.DataFrame: Filtered dataframe
+    """
+
+    title_font = "Arial"
+    body_font = "Arial"
+    title_size = 32
+    colors = ["red", "green", "blue"]
+    interpretation = False
+    extract_docx = False
+    title = "My Chart"
+    regex = ".*"
+    img_path = 'default_image.png'
+
+
+    #try:
+    #    modify = st.checkbox("Add filters on raw data")
+    #except:
+    #    try:
+    #        modify = st.checkbox("Add filters on processed data")
+    #    except:
+    #        try:
+    #            modify = st.checkbox("Add filters on parsed data")
+    #        except:
+    #            pass
+
+    #if not modify:
+    #    return df
+
+    df_ = df.copy()
+    # Try to convert datetimes into a standard format (datetime, no timezone)
+
+#modification_container = st.container()
+
+#with modification_container:
+    to_filter_columns = st.multiselect("Filter dataframe on", df_.columns)
+
+    date_column = None
+    filtered_columns = []
+
+    for column in to_filter_columns:
+        left, right = st.columns((1, 20))
+        # Treat columns with < 200 unique values as categorical if not date or numeric
+        if is_categorical_dtype(df_[column]) or (df_[column].nunique() < 120 and not is_datetime64_any_dtype(df_[column]) and not is_numeric_dtype(df_[column])):
+            user_cat_input = right.multiselect(
+                f"Values for {column}",
+                df_[column].value_counts().index.tolist(),
+                default=list(df_[column].value_counts().index)
+            )
+            df_ = df_[df_[column].isin(user_cat_input)]
+            filtered_columns.append(column)
+
+            with st.status(f"Category Distribution: {column}", expanded=False) as stat:
+                st.pyplot(plot_treemap(df_, column))
+
+        elif is_numeric_dtype(df_[column]):
+            _min = float(df_[column].min())
+            _max = float(df_[column].max())
+            step = (_max - _min) / 100
+            user_num_input = right.slider(
+                f"Values for {column}",
+                min_value=_min,
+                max_value=_max,
+                value=(_min, _max),
+                step=step,
+            )
+            df_ = df_[df_[column].between(*user_num_input)]
+            filtered_columns.append(column)
+
+            # Chart_GPT = ChartGPT(df_, title_font, body_font, title_size,
+            #      colors, interpretation, extract_docx, img_path)
+
+            with st.status(f"Numerical Distribution: {column}", expanded=False) as stat_:
+                st.pyplot(plot_hist(df_, column, bins=int(round(len(df_[column].unique())-1)/2)))
+
+        elif is_object_dtype(df_[column]):
+            try:
+                df_[column] = pd.to_datetime(df_[column], infer_datetime_format=True, errors='coerce')
+            except Exception:
+                try:
+                    df_[column] = df_[column].apply(parser.parse)
+                except Exception:
+                    pass
+
+            if is_datetime64_any_dtype(df_[column]):
+                df_[column] = df_[column].dt.tz_localize(None)
+                min_date = df_[column].min().date()
+                max_date = df_[column].max().date()
+                user_date_input = right.date_input(
+                    f"Values for {column}",
+                    value=(min_date, max_date),
+                    min_value=min_date,
+                    max_value=max_date,
+                )
+                # if len(user_date_input) == 2:
+                #     start_date, end_date = user_date_input
+                #     df_ = df_.loc[df_[column].dt.date.between(start_date, end_date)]
+                if len(user_date_input) == 2:
+                    user_date_input = tuple(map(pd.to_datetime, user_date_input))
+                    start_date, end_date = user_date_input
+                    df_ = df_.loc[df_[column].between(start_date, end_date)]
+
+                date_column = column
+
+                if date_column and filtered_columns:
+                    numeric_columns = [col for col in filtered_columns if is_numeric_dtype(df_[col])]
+                    if numeric_columns:
+                        fig = plot_line(df_, date_column, numeric_columns)
+                        #st.pyplot(fig)
+                    # now to deal with categorical columns
+                    categorical_columns = [col for col in filtered_columns if is_categorical_dtype(df_[col])]
+                    if categorical_columns:
+                        fig2 = plot_bar(df_, date_column, categorical_columns[0])
+                        #st.pyplot(fig2)
+                    with st.status(f"Date Distribution: {column}", expanded=False) as stat:
+                        try:
+                            st.pyplot(fig)
+                        except Exception as e:
+                            st.error(f"Error plotting line chart: {e}")
+                            pass
+                        try:
+                            st.pyplot(fig2)
+                        except Exception as e:
+                            st.error(f"Error plotting bar chart: {e}")
+
+
+        else:
+            user_text_input = right.text_input(
+                f"Substring or regex in {column}",
+            )
+            if user_text_input:
+                df_ = df_[df_[column].astype(str).str.contains(user_text_input)]
+    # write len of df after filtering with % of original
+    st.write(f"{len(df_)} rows ({len(df_) / len(df) * 100:.2f}%)")
+    return df_
+
+
+def get_stations():
+    base_url = 'https://imag-data.bgs.ac.uk:/GIN_V1/GINServices?Request=GetCapabilities&format=json' 
+    response = requests.get(base_url)
+    data = response.json()
+    dataframe_stations = pd.DataFrame.from_dict(data['ObservatoryList'])
+    return dataframe_stations
+
+def get_haversine_distance(lat1, lon1, lat2, lon2):
+    R = 6371
+    dlat = math.radians(lat2 - lat1)
+    dlon = math.radians(lon2 - lon1)
+    a = math.sin(dlat/2) * math.sin(dlat/2) + math.cos(math.radians(lat1)) * math.cos(math.radians(lat2)) * math.sin(dlon/2) * math.sin(dlon/2)
+    c = 2 * math.atan2(math.sqrt(a), math.sqrt(1-a))
+    d = R * c
+    return d
+
+def compare_stations(test_lat_lon, data_table, distance=1000, closest=False):
+    table_updated = pd.DataFrame()
+    distances = dict()
+    for lat,lon,names in data_table[['Latitude', 'Longitude', 'Name']].values:
+        harv_distance = get_haversine_distance(test_lat_lon[0], test_lat_lon[1], lat, lon)
+        if harv_distance < distance:
+            #print(f"Station {names} is at {round(harv_distance,2)} km from the test point")
+            table_updated = pd.concat([table_updated, data_table[data_table['Name'] == names]])
+            distances[names] = harv_distance
+    if closest:
+        closest_station = min(distances, key=distances.get)
+        #print(f"The closest station is {closest_station} at {round(distances[closest_station],2)} km")
+        table_updated = data_table[data_table['Name'] == closest_station]
+        table_updated['Distance'] = distances[closest_station]
+    return table_updated
+
+def get_data(IagaCode, start_date, end_date):
+    try:
+        start_date_ = datetime.datetime.strptime(start_date, '%Y-%m-%d')
+    except ValueError as e:
+        print(f"Error: {e}")
+        start_date_ = pd.to_datetime(start_date)
+    try:
+        end_date_ = datetime.datetime.strptime(end_date, '%Y-%m-%d')
+    except ValueError as e:
+        print(f"Error: {e}")
+        end_date_ = pd.to_datetime(end_date)
+
+    duration = end_date_ - start_date_
+    # Define the parameters for the request
+    params = {
+        'Request': 'GetData',
+        'format': 'PNG',
+        'testObsys': '0',
+        'observatoryIagaCode': IagaCode,
+        'samplesPerDay': 'minute',
+        'publicationState': 'Best available',
+        'dataStartDate': start_date,
+        # make substraction
+        'dataDuration': duration.days,
+        'traceList': '1234',
+        'colourTraces': 'true',
+        'pictureSize': 'Automatic',
+        'dataScale': 'Automatic',
+        'pdfSize': '21,29.7',
+    }
+
+    base_url_json = 'https://imag-data.bgs.ac.uk:/GIN_V1/GINServices?Request=GetData&format=json'
+    #base_url_img = 'https://imag-data.bgs.ac.uk:/GIN_V1/GINServices?Request=GetData&format=png'
+
+    for base_url in [base_url_json]:#, base_url_img]:
+        response = requests.get(base_url, params=params)
+        if response.status_code == 200:
+            content_type = response.headers.get('Content-Type')
+            if 'image' in content_type:
+                # f"custom_plot_{new_dataset.iloc[0]['IagaCode']}_{str_date.replace(':', '_')}.png"
+                # output_image_path = "plot_image.png" 
+                # with open(output_image_path, 'wb') as file:
+                #     file.write(response.content)
+                # print(f"Image successfully saved as {output_image_path}")
+                
+                # # Display the image
+                # img = mpimg.imread(output_image_path)
+                # plt.imshow(img)
+                # plt.axis('off')  # Hide axes
+                # plt.show()
+                # img_answer = Image.open(output_image_path)
+                img_answer = None
+            else:
+                print(f"Unexpected content type: {content_type}")
+                #print("Response content:")
+                #print(response.content.decode('utf-8'))  # Attempt to print response as text
+                # return json
+                answer = response.json()
+        else:
+            print(f"Failed to retrieve data. HTTP Status code: {response.status_code}")
+            print("Response content:")
+            print(response.content.decode('utf-8'))
+    return answer#, img_answer
+
+
+# def get_data(IagaCode, start_date, end_date):
+#     # Convert dates to datetime
+#     try:
+#         start_date_ = pd.to_datetime(start_date)
+#         end_date_ = pd.to_datetime(end_date)
+#     except ValueError as e:
+#         print(f"Error: {e}")
+#         return None, None
+
+#     duration = (end_date_ - start_date_).days
+
+#     # Define the parameters for the request
+#     params = {
+#         'Request': 'GetData',
+#         'format': 'json',
+#         'testObsys': '0',
+#         'observatoryIagaCode': IagaCode,
+#         'samplesPerDay': 'minute',
+#         'publicationState': 'Best available',
+#         'dataStartDate': start_date_.strftime('%Y-%m-%d'),
+#         'dataDuration': duration,
+#         'traceList': '1234',
+#         'colourTraces': 'true',
+#         'pictureSize': 'Automatic',
+#         'dataScale': 'Automatic',
+#         'pdfSize': '21,29.7',
+#     }
+
+#     base_url_json = 'https://imag-data.bgs.ac.uk:/GIN_V1/GINServices?Request=GetData&format=json'
+#     base_url_img = 'https://imag-data.bgs.ac.uk:/GIN_V1/GINServices?Request=GetData&format=png'
+
+#     try:
+#         # Request JSON data
+#         response_json = requests.get(base_url_json, params=params)
+#         response_json.raise_for_status()  # Raises an error for bad status codes
+#         data = response_json.json()
+
+#         # Request Image
+#         params['format'] = 'png'
+#         response_img = requests.get(base_url_img, params=params)
+#         response_img.raise_for_status()
+
+#         # Save and display image if response is successful
+#         if 'image' in response_img.headers.get('Content-Type'):
+#             output_image_path = "plot_image.png"
+#             with open(output_image_path, 'wb') as file:
+#                 file.write(response_img.content)
+#             print(f"Image successfully saved as {output_image_path}")
+
+#             img = mpimg.imread(output_image_path)
+#             plt.imshow(img)
+#             plt.axis('off')
+#             plt.show()
+#             img_answer = Image.open(output_image_path)
+#         else:
+#             img_answer = None
+
+#         return data, img_answer
+
+#     except requests.RequestException as e:
+#         print(f"Request failed: {e}")
+#         return None, None
+#     except ValueError as e:
+#         print(f"JSON decode error: {e}")
+#         return None, None
+
+def clean_uap_data(dataset, lat, lon, date):
+    # Assuming 'nuforc' is already defined
+    processed = dataset[dataset[[lat, lon, date]].notnull().all(axis=1)]
+    # Converting 'Lat' and 'Long' columns to floats, handling errors
+    processed[lat] = pd.to_numeric(processed[lat], errors='coerce')
+    processed[lon] = pd.to_numeric(processed[lon], errors='coerce')
+
+    # if processed[date].min() < pd.to_datetime('1677-09-22'):
+    #     processed.loc[processed[date] < pd.to_datetime('1677-09-22'), 'corrected_date'] = pd.to_datetime('1677-09-22 00:00:00')
+
+    procesed = processed[processed[date] >= '1677-09-22']
+
+    # convert date to str
+    #processed[date] = processed[date].astype(str)
+    # Dropping rows where 'Lat' or 'Long' conversion failed (i.e., became NaN)
+    processed = processed.dropna(subset=[lat, lon])
+    return processed
+
+
+def plot_overlapped_timeseries(data_list, event_times, window_hours=12, save_path=None):
+    fig, axs = plt.subplots(4, 1, figsize=(12, 16), sharex=True)
+    fig.patch.set_alpha(0)  # Make figure background transparent
+
+    components = ['X', 'Y', 'Z', 'S']
+    colors = ['red', 'green', 'blue', 'black']
+
+    for i, component in enumerate(components):
+        axs[i].patch.set_alpha(0)  # Make subplot background transparent
+        axs[i].set_ylabel(component, color='orange')
+        axs[i].grid(True, color='orange', alpha=0.3)
+        
+        for spine in axs[i].spines.values():
+            spine.set_color('orange')
+        
+        axs[i].tick_params(axis='both', colors='orange')  # Change tick color
+        axs[i].set_title(f'{component}', color='orange')
+        axs[i].set_xlabel('Time Difference from Event (hours)', color='orange')
+
+        for j, (df, event_time) in enumerate(zip(data_list, event_times)):
+            # Convert datetime column to UTC if it has timezone info, otherwise assume it's UTC
+            df['datetime'] = pd.to_datetime(df['datetime']).dt.tz_localize(None)
+            
+            # Convert event_time to UTC if it has timezone info, otherwise assume it's UTC
+            event_time = pd.to_datetime(event_time).tz_localize(None)
+            
+            # Calculate time difference from event
+            df['time_diff'] = (df['datetime'] - event_time).dt.total_seconds() / 3600  # Convert to hours
+            
+            # Filter data within the specified window
+            df_window = df[(df['time_diff'] >= -window_hours) & (df['time_diff'] <= window_hours)]
+
+            # normalize component data
+            df_window[component] = (df_window[component] - df_window[component].mean()) / df_window[component].std()
+            
+            axs[i].plot(df_window['time_diff'], df_window[component], color=colors[i], alpha=0.7, label=f'Event {j+1}', linewidth=1)
+        
+        axs[i].axvline(x=0, color='red', linewidth=2, linestyle='--', label='Event Time')
+        axs[i].set_xlim(-window_hours, window_hours)
+        #axs[i].legend(loc='upper left', bbox_to_anchor=(1, 1))
+
+    axs[-1].set_xlabel('Hours from Event', color='orange')
+    fig.suptitle('Overlapped Time Series of Components', fontsize=16, color='orange')
+    
+    plt.tight_layout()
+    plt.subplots_adjust(top=0.95, right=0.85)
+
+    if save_path:
+        fig.savefig(save_path, transparent=True, bbox_inches='tight')
+        plt.close(fig)
+        return save_path
+    else:
+        return fig
+    
+def plot_average_timeseries(data_list, event_times, window_hours=12, save_path=None):
+    fig, axs = plt.subplots(4, 1, figsize=(12, 16), sharex=True)
+    fig.patch.set_alpha(0)  # Make figure background transparent
+
+    components = ['X', 'Y', 'Z', 'S']
+    colors = ['red', 'green', 'blue', 'black']
+
+    for i, component in enumerate(components):
+        axs[i].patch.set_alpha(0)
+        axs[i].set_ylabel(component, color='orange')
+        axs[i].grid(True, color='orange', alpha=0.3)
+
+        for spine in axs[i].spines.values():
+                spine.set_color('orange')
+        
+        axs[i].tick_params(axis='both', colors='orange')
+
+        all_data = []
+        time_diffs = []
+
+        for j, (df, event_time) in enumerate(zip(data_list, event_times)):
+            # Convert datetime column to UTC if it has timezone info, otherwise assume it's UTC
+            df['datetime'] = pd.to_datetime(df['datetime']).dt.tz_localize(None)
+
+            # Convert event_time to UTC if it has timezone info, otherwise assume it's UTC
+            event_time = pd.to_datetime(event_time).tz_localize(None)
+
+            # Calculate time difference from event
+            df['time_diff'] = (df['datetime'] - event_time).dt.total_seconds() / 3600  # Convert to hours
+
+            # Filter data within the specified window
+            df_window = df[(df['time_diff'] >= -window_hours) & (df['time_diff'] <= window_hours)]
+
+            # Normalize component data
+            df_window[component] = (df_window[component] - df_window[component].mean())# / df_window[component].std()
+
+            all_data.append(df_window[component].values)
+            time_diffs.append(df_window['time_diff'].values)
+
+        # Calculate average and standard deviation
+        try:
+            avg_data = np.mean(all_data, axis=0)
+        except:
+            avg_data = np.zeros_like(all_data[0])
+        try:
+            std_data = np.std(all_data, axis=0)
+        except:
+            std_data = np.zeros_like(avg_data)
+
+        axs[-1].set_xlabel('Hours from Event', color='orange')
+        fig.suptitle('Average Time Series of Components', fontsize=16, color='orange')
+
+        # Plot average line
+        axs[i].plot(time_diffs[0], avg_data, color=colors[i], label='Average')
+
+        # Plot standard deviation as shaded region
+        try:
+            axs[i].fill_between(time_diffs[0], avg_data - std_data, avg_data + std_data, color=colors[i], alpha=0.2)
+        except:
+            pass
+
+        axs[i].axvline(x=0, color='red', linewidth=2, linestyle='--', label='Event Time')
+        axs[i].set_xlim(-window_hours, window_hours)
+        # orange frame, orange label legend
+        axs[i].legend(loc='upper right', bbox_to_anchor=(1, 1), facecolor='black', framealpha=.4, labelcolor='orange', edgecolor='orange')
+
+    plt.tight_layout()
+    plt.subplots_adjust(top=0.95, right=0.85)
+
+    if save_path:
+        fig.savefig(save_path, transparent=True, bbox_inches='tight')
+        plt.close(fig)
+        return save_path
+    else:
+        return fig
+    
+def align_series(reference, series):
+    reference = reference.flatten()
+    series = series.flatten()
+    _, path = fastdtw(reference, series, dist=euclidean)
+    aligned = np.zeros(len(reference))
+    for ref_idx, series_idx in path:
+        aligned[ref_idx] = series[series_idx]
+    return aligned
+
+def plot_average_timeseries_with_dtw(data_list, event_times, window_hours=12, save_path=None):
+    fig, axs = plt.subplots(4, 1, figsize=(12, 16), sharex=True)
+    fig.patch.set_alpha(0)  # Make figure background transparent
+
+    components = ['X', 'Y', 'Z', 'S']
+    colors = ['red', 'green', 'blue', 'black']
+    fig.text(0.02, 0.5, 'Geomagnetic Variation (nT)', va='center', rotation='vertical', color='orange')
+
+
+    for i, component in enumerate(components):
+        axs[i].patch.set_alpha(0)
+        axs[i].set_ylabel(component, color='orange', rotation=90)
+        axs[i].grid(True, color='orange', alpha=0.3)
+        
+        for spine in axs[i].spines.values():
+            spine.set_color('orange')
+        
+        axs[i].tick_params(axis='both', colors='orange')
+
+        all_aligned_data = []
+        reference_df = None
+
+        for j, (df, event_time) in enumerate(zip(data_list, event_times)):
+            df['datetime'] = pd.to_datetime(df['datetime']).dt.tz_localize(None)
+            event_time = pd.to_datetime(event_time).tz_localize(None)
+            df['time_diff'] = (df['datetime'] - event_time).dt.total_seconds() / 3600
+            df_window = df[(df['time_diff'] >= -window_hours) & (df['time_diff'] <= window_hours)]
+            df_window[component] = (df_window[component] - df_window[component].mean())# / df_window[component].std()
+            
+            if reference_df is None:
+                reference_df = df_window
+                all_aligned_data.append(reference_df[component].values)
+            else:
+                try:
+                    aligned_series = align_series(reference_df[component].values, df_window[component].values)
+                    all_aligned_data.append(aligned_series)
+                except:
+                    pass
+
+        # Calculate average and standard deviation of aligned data
+        all_aligned_data = np.array(all_aligned_data)
+        avg_data = np.mean(all_aligned_data, axis=0)
+
+        # round float to avoid sqrt errors
+        def calculate_std(data):
+            if data is not None and len(data) > 0:
+                data = np.array(data)
+                std_data = np.std(data)
+                return std_data
+            else:
+                return "Data is empty or not a list"
+            
+        std_data = calculate_std(all_aligned_data)
+
+        # Plot average line
+        axs[i].plot(reference_df['time_diff'], avg_data, color=colors[i], label='Average')
+
+        # Plot standard deviation as shaded region
+        try:
+            axs[i].fill_between(reference_df['time_diff'], avg_data - std_data, avg_data + std_data, color=colors[i], alpha=0.2)
+        except TypeError as e:
+            #print(f"Error: {e}")
+            pass
+            
+
+        axs[i].axvline(x=0, color='red', linewidth=2, linestyle='--', label='Event Time')
+        axs[i].set_xlim(-window_hours, window_hours)
+        axs[i].legend(loc='upper right', bbox_to_anchor=(1, 1), facecolor='black', framealpha=.2, labelcolor='orange', edgecolor='orange')
+
+
+    axs[-1].set_xlabel('Hours from Event', color='orange')
+    fig.suptitle('Average Time Series of Components (FastDTW Aligned)', fontsize=16, color='orange')
+
+    plt.tight_layout()
+    plt.subplots_adjust(top=0.85, right=0.85, left=0.1)
+
+    if save_path:
+        fig.savefig(save_path, transparent=True, bbox_inches='tight')
+        plt.close(fig)
+        return save_path
+    else:
+        return fig
+
+def plot_data_custom(df, date, save_path=None, subtitle=None):
+    df['datetime'] = pd.to_datetime(df['datetime'])
+    event = pd.to_datetime(date)
+    window = timedelta(hours=12)
+    x_min = event - window
+    x_max = event + window
+
+    fig, axs = plt.subplots(4, 1, figsize=(12, 12), sharex=True)
+    fig.patch.set_alpha(0)  # Make figure background transparent
+
+    components = ['X', 'Y', 'Z', 'S']
+    colors = ['red', 'green', 'blue', 'black']
+
+    fig.text(0.02, 0.5, 'Geomagnetic Variation (nT)', va='center', rotation='vertical', color='orange')
+
+    # if df[component].isnull().all().all():
+    #     return None
+            
+    for i, component in enumerate(components):
+        axs[i].plot(df['datetime'], df[component], label=component, color=colors[i])
+        axs[i].axvline(x=event, color='red', linewidth=2, label='Event', linestyle='--')
+        axs[i].set_ylabel(component, color='orange', rotation=90)
+        axs[i].set_xlim(x_min, x_max)
+        axs[i].legend(loc='upper right', bbox_to_anchor=(1, 1), facecolor='black', framealpha=.2, labelcolor='orange', edgecolor='orange')
+        axs[i].grid(True, color='orange', alpha=0.3)
+        axs[i].patch.set_alpha(0)  # Make subplot background transparent
+        
+        for spine in axs[i].spines.values():
+            spine.set_color('orange')
+        
+        axs[i].xaxis.set_major_formatter(mdates.DateFormatter('%H:%M'))
+        axs[i].xaxis.set_major_locator(mdates.HourLocator(interval=1))
+        axs[i].tick_params(axis='both', colors='orange')
+
+    plt.setp(axs[-1].xaxis.get_majorticklabels(), rotation=45)
+    axs[-1].set_xlabel('Hours', color='orange')
+    fig.suptitle(f'Time Series of Components with Event Marks\n{subtitle}', fontsize=12, color='orange')
+    
+    plt.tight_layout()
+    #plt.subplots_adjust(top=0.85)
+    plt.subplots_adjust(top=0.85, right=0.85, left=0.1)
+
+
+    if save_path:
+        fig.savefig(save_path, transparent=True)
+        plt.close(fig)
+        return save_path
+    else:
+        return fig
+
+
+def batch_requests(stations, dataset, lon, lat, date, distance=100):
+    results = {"station": [], "data": [], "image": [], "custom_image": []}
+    all_data = []
+    all_event_times = []
+
+    for lon_, lat_, date_ in dataset[[lon, lat, date]].values:
+        test_lat_lon = (lat_, lon_)
+        try:
+            str_date = pd.to_datetime(date_).strftime('%Y-%m-%dT%H:%M:%S')
+        except:
+            str_date = date_
+        twelve_hours = pd.Timedelta(hours=12)
+        forty_eight_hours = pd.Timedelta(hours=48)
+        try:
+            str_date_start = (pd.to_datetime(str_date) - twelve_hours).strftime('%Y-%m-%dT%H:%M:%S')
+            str_date_end = (pd.to_datetime(str_date) + forty_eight_hours).strftime('%Y-%m-%dT%H:%M:%S')
+        except Exception as e:
+            print(f"Error: {e}")
+            pass
+        
+        try:
+            new_dataset = compare_stations(test_lat_lon, stations, distance=distance, closest=True)
+            station_name = new_dataset['Name']
+            station_distance = new_dataset['Distance']
+            test_ = get_data(new_dataset.iloc[0]['IagaCode'], str_date_start, str_date_end)
+
+            if test_:
+                results["station"].append(new_dataset.iloc[0]['IagaCode'])
+                results["data"].append(test_)
+                plotted = pd.DataFrame({
+                    'datetime': test_['datetime'],
+                    'X': test_['X'],
+                    'Y': test_['Y'],
+                    'Z': test_['Z'],
+                    'S': test_['S'],
+                })
+                all_data.append(plotted)
+                all_event_times.append(pd.to_datetime(date_))
+                # print(date_)
+                additional_data = f"Date: {date_}\nLat/Lon: {lat_}, {lon_}\nClosest station: {station_name.values[0]}\n Distance:{round(station_distance.values[0],2)} km"
+                fig = plot_data_custom(plotted, date=pd.to_datetime(date_), save_path=None, subtitle =additional_data)
+                with st.status(f'Magnetic Data: {date_}', expanded=False) as status:
+                    st.pyplot(fig)
+                    status.update(f'Magnetic Data: {date_} - Finished!')
+        except Exception as e:
+            #print(f"An error occurred: {e}")
+            pass
+
+    if all_data:
+        fig_overlapped = plot_overlapped_timeseries(all_data, all_event_times)
+        display(fig_overlapped)
+        plt.close(fig_overlapped)
+        # fig_average = plot_average_timeseries(all_data, all_event_times)
+        # st.pyplot(fig_average)
+        fig_average_aligned = plot_average_timeseries_with_dtw(all_data, all_event_times)
+        with st.status(f'Dynamic Time Warping Data', expanded=False) as stts:
+            st.pyplot(fig_average_aligned)
+    return results
+
+
+df = pd.DataFrame()
+
+
+# Upload dataset
+uploaded_file = st.file_uploader("Choose a file", type=["csv", "xlsx"])
+
+if uploaded_file is not None:
+    if uploaded_file.name.endswith('.csv'):
+        df = pd.read_csv(uploaded_file)
+    else:
+        df = pd.read_excel(uploaded_file)
+    stations = get_stations()
+    st.write("Dataset Loaded:")
+    df = filter_dataframe(df)
+    st.dataframe(df)
+
+    # Select columns
+    lon_col = st.selectbox("Select Longitude Column", df.columns)
+    lat_col = st.selectbox("Select Latitude Column", df.columns)
+    date_col = st.selectbox("Select Date Column", df.columns)
+    distance = st.number_input("Enter Distance", min_value=0, value=100)
+
+    # Process data
+    if st.button("Process Data"):
+        cases = clean_uap_data(df, lat_col, lon_col, date_col)
+        results = batch_requests(stations, cases, lon_col, lat_col, date_col, distance=distance)

map.py

@@ -0,0 +1,506 @@
+import json
+import streamlit as st
+#import geopandas as gpd
+from keplergl import keplergl
+import pandas as pd
+import streamlit as st
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import seaborn as sns
+from uap_analyzer import UAPParser, UAPAnalyzer, UAPVisualizer
+# import ChartGen
+# from ChartGen import ChartGPT
+from Levenshtein import distance
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import confusion_matrix
+from stqdm import stqdm
+stqdm.pandas()
+import streamlit.components.v1 as components
+from dateutil import parser
+from sentence_transformers import SentenceTransformer
+import torch
+import squarify
+import matplotlib.colors as mcolors
+import textwrap
+import datamapplot
+from streamlit_extras.stateful_button import button as stateful_button
+from streamlit_keplergl import keplergl_static
+from keplergl import KeplerGl
+
+
+st.set_option('deprecation.showPyplotGlobalUse', False)
+
+from pandas.api.types import (
+    is_categorical_dtype,
+    is_datetime64_any_dtype,
+    is_numeric_dtype,
+    is_object_dtype,
+)
+
+st.title('Interactive Map')
+
+# Initialize session state
+if 'analyzers' not in st.session_state:
+    st.session_state['analyzers'] = []
+if 'col_names' not in st.session_state:
+    st.session_state['col_names'] = []
+if 'clusters' not in st.session_state:
+    st.session_state['clusters'] = {}
+if 'new_data' not in st.session_state:
+    st.session_state['new_data'] = pd.DataFrame()
+if 'dataset' not in st.session_state:
+    st.session_state['dataset'] = pd.DataFrame()
+if 'data_processed' not in st.session_state:
+    st.session_state['data_processed'] = False
+if 'stage' not in st.session_state:
+    st.session_state['stage'] = 0
+if 'filtered_data' not in st.session_state:
+    st.session_state['filtered_data'] = None
+if 'gemini_answer' not in st.session_state:
+    st.session_state['gemini_answer'] = None
+if 'parsed_responses' not in st.session_state:
+    st.session_state['parsed_responses'] = None
+if 'map_generated' not in st.session_state:
+    st.session_state['map_generated'] = False
+if 'date_loaded' not in st.session_state:
+    st.session_state['data_loaded'] = False
+
+
+if "datasets" not in st.session_state:
+    st.session_state.datasets = []
+
+# sf_zip_geo_gdf = gpd.read_file("sf_zip_geo.geojson")
+# sf_zip_geo_gdf.label = "SF Zip Geo"
+# sf_zip_geo_gdf.id = "sf-zip-geo"
+# st.session_state.datasets.append(sf_zip_geo_gdf)
+
+def plot_treemap(df, column, top_n=32):
+        # Get the value counts and the top N labels
+        value_counts = df[column].value_counts()
+        top_labels = value_counts.iloc[:top_n].index
+        
+        # Use np.where to replace all values not in the top N with 'Other'
+        revised_column = f'{column}_revised'
+        df[revised_column] = np.where(df[column].isin(top_labels), df[column], 'Other')
+
+        # Get the value counts including the 'Other' category
+        sizes = df[revised_column].value_counts().values
+        labels = df[revised_column].value_counts().index
+
+        # Get a gradient of colors
+        # colors = list(mcolors.TABLEAU_COLORS.values())
+
+        n_colors = len(sizes)
+        colors = plt.cm.Oranges(np.linspace(0.3, 0.9, n_colors))[::-1]
+
+
+        # Get % of each category
+        percents = sizes / sizes.sum()
+
+        # Prepare labels with percentages
+        labels = [f'{label}\n {percent:.1%}' for label, percent in zip(labels, percents)]
+
+        fig, ax = plt.subplots(figsize=(20, 12))
+
+        # Plot the treemap
+        squarify.plot(sizes=sizes, label=labels, alpha=0.7, pad=True, color=colors, text_kwargs={'fontsize': 10})
+
+        ax = plt.gca()
+        # Iterate over text elements and rectangles (patches) in the axes for color adjustment
+        for text, rect in zip(ax.texts, ax.patches):
+            background_color = rect.get_facecolor()
+            r, g, b, _ = mcolors.to_rgba(background_color)
+            brightness = np.average([r, g, b])
+            text.set_color('white' if brightness < 0.5 else 'black')
+
+            # Adjust font size based on rectangle's area and wrap long text
+            coef = 0.8
+            font_size = np.sqrt(rect.get_width() * rect.get_height()) * coef
+            text.set_fontsize(font_size)
+            wrapped_text = textwrap.fill(text.get_text(), width=20)
+            text.set_text(wrapped_text)
+
+        plt.axis('off')
+        plt.gca().invert_yaxis()
+        plt.gcf().set_size_inches(20, 12)
+
+        fig.patch.set_alpha(0)
+
+        ax.patch.set_alpha(0)
+        return fig
+
+def plot_hist(df, column, bins=10, kde=True):
+        fig, ax = plt.subplots(figsize=(12, 6))
+        sns.histplot(data=df, x=column, kde=True, bins=bins,color='orange')
+        # set the ticks and frame in orange
+        ax.spines['bottom'].set_color('orange')
+        ax.spines['top'].set_color('orange')
+        ax.spines['right'].set_color('orange')
+        ax.spines['left'].set_color('orange')
+        ax.xaxis.label.set_color('orange')
+        ax.yaxis.label.set_color('orange')
+        ax.tick_params(axis='x', colors='orange')
+        ax.tick_params(axis='y', colors='orange')
+        ax.title.set_color('orange')
+
+        # Set transparent background
+        fig.patch.set_alpha(0)
+        ax.patch.set_alpha(0)
+        return fig
+
+
+def plot_line(df, x_column, y_columns, figsize=(12, 10), color='orange', title=None, rolling_mean_value=2):
+    import matplotlib.cm as cm
+    # Sort the dataframe by the date column
+    df = df.sort_values(by=x_column)
+
+    # Calculate rolling mean for each y_column
+    if rolling_mean_value:
+        df[y_columns] = df[y_columns].rolling(len(df) // rolling_mean_value).mean()
+
+    # Create the plot
+    fig, ax = plt.subplots(figsize=figsize)
+
+    colors = cm.Oranges(np.linspace(0.2, 1, len(y_columns)))
+
+    # Plot each y_column as a separate line with a different color
+    for i, y_column in enumerate(y_columns):
+        df.plot(x=x_column, y=y_column, ax=ax, color=colors[i], label=y_column, linewidth=.5)
+
+    # Rotate x-axis labels
+    ax.set_xticklabels(ax.get_xticklabels(), rotation=30, ha='right')
+
+    # Format x_column as date if it is
+    if np.issubdtype(df[x_column].dtype, np.datetime64) or np.issubdtype(df[x_column].dtype, np.timedelta64):
+        df[x_column] = pd.to_datetime(df[x_column]).dt.date
+
+    # Set title, labels, and legend
+    ax.set_title(title or f'{", ".join(y_columns)} over {x_column}', color=color, fontweight='bold')
+    ax.set_xlabel(x_column, color=color)
+    ax.set_ylabel(', '.join(y_columns), color=color)
+    ax.spines['bottom'].set_color('orange')
+    ax.spines['top'].set_color('orange')
+    ax.spines['right'].set_color('orange')
+    ax.spines['left'].set_color('orange')
+    ax.xaxis.label.set_color('orange')
+    ax.yaxis.label.set_color('orange')
+    ax.tick_params(axis='x', colors='orange')
+    ax.tick_params(axis='y', colors='orange')
+    ax.title.set_color('orange')
+
+    ax.legend(loc='upper right', bbox_to_anchor=(1, 1), facecolor='black', framealpha=.4, labelcolor='orange', edgecolor='orange')
+
+    # Remove background
+    fig.patch.set_alpha(0)
+    ax.patch.set_alpha(0)
+
+    return fig
+
+def plot_bar(df, x_column, y_column, figsize=(12, 10), color='orange', title=None):
+    fig, ax = plt.subplots(figsize=figsize)
+
+    sns.barplot(data=df, x=x_column, y=y_column, color=color, ax=ax)
+
+    ax.set_title(title if title else f'{y_column} by {x_column}', color=color, fontweight='bold')
+    ax.set_xlabel(x_column, color=color)
+    ax.set_ylabel(y_column, color=color)
+
+    ax.tick_params(axis='x', colors=color)
+    ax.tick_params(axis='y', colors=color)
+
+    # Remove background
+    fig.patch.set_alpha(0)
+    ax.patch.set_alpha(0)
+    ax.spines['bottom'].set_color('orange')
+    ax.spines['top'].set_color('orange')
+    ax.spines['right'].set_color('orange')
+    ax.spines['left'].set_color('orange')
+    ax.xaxis.label.set_color('orange')
+    ax.yaxis.label.set_color('orange')
+    ax.tick_params(axis='x', colors='orange')
+    ax.tick_params(axis='y', colors='orange')
+    ax.title.set_color('orange')
+    ax.legend(loc='upper right', bbox_to_anchor=(1, 1), facecolor='black', framealpha=.4, labelcolor='orange', edgecolor='orange')
+
+    return fig
+
+def plot_grouped_bar(df, x_columns, y_column, figsize=(12, 10), colors=None, title=None):
+    fig, ax = plt.subplots(figsize=figsize)
+
+    width = 0.8 / len(x_columns)  # the width of the bars
+    x = np.arange(len(df))  # the label locations
+
+    for i, x_column in enumerate(x_columns):
+        sns.barplot(data=df, x=x, y=y_column, color=colors[i] if colors else None, ax=ax, width=width, label=x_column)
+        x += width  # add the width of the bar to the x position for the next bar
+
+    ax.set_title(title if title else f'{y_column} by {", ".join(x_columns)}', color='orange', fontweight='bold')
+    ax.set_xlabel('Groups', color='orange')
+    ax.set_ylabel(y_column, color='orange')
+
+    ax.set_xticks(x - width * len(x_columns) / 2)
+    ax.set_xticklabels(df.index)
+
+    ax.tick_params(axis='x', colors='orange')
+    ax.tick_params(axis='y', colors='orange')
+
+    # Remove background
+    fig.patch.set_alpha(0)
+    ax.patch.set_alpha(0)
+    ax.spines['bottom'].set_color('orange')
+    ax.spines['top'].set_color('orange')
+    ax.spines['right'].set_color('orange')
+    ax.spines['left'].set_color('orange')
+    ax.xaxis.label.set_color('orange')
+    ax.yaxis.label.set_color('orange')
+    ax.title.set_color('orange')
+    ax.legend(loc='upper right', bbox_to_anchor=(1, 1), facecolor='black', framealpha=.4, labelcolor='orange', edgecolor='orange')
+
+    return fig
+
+def generate_kepler_map(data):
+    map_config = keplergl(data, height=400)
+    return map_config
+
+def filter_dataframe(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Adds a UI on top of a dataframe to let viewers filter columns
+
+    Args:
+        df (pd.DataFrame): Original dataframe
+
+    Returns:
+        pd.DataFrame: Filtered dataframe
+    """
+
+    title_font = "Arial"
+    body_font = "Arial"
+    title_size = 32
+    colors = ["red", "green", "blue"]
+    interpretation = False
+    extract_docx = False
+    title = "My Chart"
+    regex = ".*"
+    img_path = 'default_image.png'
+
+
+    #try:
+    #    modify = st.checkbox("Add filters on raw data")
+    #except:
+    #    try:
+    #        modify = st.checkbox("Add filters on processed data")
+    #    except:
+    #        try:
+    #            modify = st.checkbox("Add filters on parsed data")
+    #        except:
+    #            pass
+
+    #if not modify:
+    #    return df
+
+    df_ = df.copy()
+    # Try to convert datetimes into a standard format (datetime, no timezone)
+
+#modification_container = st.container()
+
+#with modification_container:
+    try:
+        to_filter_columns = st.multiselect("Filter dataframe on", df_.columns)
+    except:
+        try:
+            to_filter_columns = st.multiselect("Filter dataframe", df_.columns)
+        except:
+            try:
+                to_filter_columns = st.multiselect("Filter the dataframe on", df_.columns)
+            except:
+                pass
+
+    date_column = None
+    filtered_columns = []
+
+    for column in to_filter_columns:
+        left, right = st.columns((1, 20))
+        # Treat columns with < 200 unique values as categorical if not date or numeric
+        if is_categorical_dtype(df_[column]) or (df_[column].nunique() < 120 and not is_datetime64_any_dtype(df_[column]) and not is_numeric_dtype(df_[column])):
+            user_cat_input = right.multiselect(
+                f"Values for {column}",
+                df_[column].value_counts().index.tolist(),
+                default=list(df_[column].value_counts().index)
+            )
+            df_ = df_[df_[column].isin(user_cat_input)]
+            filtered_columns.append(column)
+
+            with st.status(f"Category Distribution: {column}", expanded=False) as stat:
+                st.pyplot(plot_treemap(df_, column))
+
+        elif is_numeric_dtype(df_[column]):
+            _min = float(df_[column].min())
+            _max = float(df_[column].max())
+            step = (_max - _min) / 100
+            user_num_input = right.slider(
+                f"Values for {column}",
+                min_value=_min,
+                max_value=_max,
+                value=(_min, _max),
+                step=step,
+            )
+            df_ = df_[df_[column].between(*user_num_input)]
+            filtered_columns.append(column)
+
+            # Chart_GPT = ChartGPT(df_, title_font, body_font, title_size,
+            #      colors, interpretation, extract_docx, img_path)
+
+            with st.status(f"Numerical Distribution: {column}", expanded=False) as stat_:
+                st.pyplot(plot_hist(df_, column, bins=int(round(len(df_[column].unique())-1)/2)))
+
+        elif is_object_dtype(df_[column]):
+            try:
+                df_[column] = pd.to_datetime(df_[column], infer_datetime_format=True, errors='coerce')
+            except Exception:
+                try:
+                    df_[column] = df_[column].apply(parser.parse)
+                except Exception:
+                    pass
+
+            if is_datetime64_any_dtype(df_[column]):
+                df_[column] = df_[column].dt.tz_localize(None)
+                min_date = df_[column].min().date()
+                max_date = df_[column].max().date()
+                user_date_input = right.date_input(
+                    f"Values for {column}",
+                    value=(min_date, max_date),
+                    min_value=min_date,
+                    max_value=max_date,
+                )
+                # if len(user_date_input) == 2:
+                #     start_date, end_date = user_date_input
+                #     df_ = df_.loc[df_[column].dt.date.between(start_date, end_date)]
+                if len(user_date_input) == 2:
+                    user_date_input = tuple(map(pd.to_datetime, user_date_input))
+                    start_date, end_date = user_date_input
+                    df_ = df_.loc[df_[column].between(start_date, end_date)]
+
+                date_column = column
+
+                if date_column and filtered_columns:
+                    numeric_columns = [col for col in filtered_columns if is_numeric_dtype(df_[col])]
+                    if numeric_columns:
+                        fig = plot_line(df_, date_column, numeric_columns)
+                        #st.pyplot(fig)
+                    # now to deal with categorical columns
+                    categorical_columns = [col for col in filtered_columns if is_categorical_dtype(df_[col])]
+                    if categorical_columns:
+                        fig2 = plot_bar(df_, date_column, categorical_columns[0])
+                        #st.pyplot(fig2)
+                    with st.status(f"Date Distribution: {column}", expanded=False) as stat:
+                        try:
+                            st.pyplot(fig)
+                        except Exception as e:
+                            st.error(f"Error plotting line chart: {e}")
+                            pass
+                        try:
+                            st.pyplot(fig2)
+                        except Exception as e:
+                            st.error(f"Error plotting bar chart: {e}")
+
+
+        else:
+            user_text_input = right.text_input(
+                f"Substring or regex in {column}",
+            )
+            if user_text_input:
+                df_ = df_[df_[column].astype(str).str.contains(user_text_input)]
+    # write len of df after filtering with % of original
+    st.write(f"{len(df_)} rows ({len(df_) / len(df) * 100:.2f}%)")
+    return df_
+
+def find_lat_lon_columns(df):
+    lat_columns = df.columns[df.columns.str.lower().str.contains('lat')]
+    lon_columns = df.columns[df.columns.str.lower().str.contains('lon|lng')]
+    
+    if len(lat_columns) > 0 and len(lon_columns) > 0:
+        return lat_columns[0], lon_columns[0]
+    else:
+        return None, None
+
+my_dataset = st.file_uploader("Upload Parsed DataFrame", type=["csv", "xlsx"])
+map_1 = KeplerGl(height=800)
+powerplant = pd.read_csv('global_power_plant_database.csv')
+secret_bases = pd.read_csv('secret_bases.csv')
+
+map_1.add_data(
+            data=secret_bases, name="secret_bases"
+        )
+map_1.add_data(
+        data=powerplant, name='nuclear_powerplants'
+        )
+
+
+if my_dataset is not None :
+    try:
+        if my_dataset.type == "text/csv":
+            data = pd.read_csv(my_dataset)
+        elif my_dataset.type == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet":
+            data = pd.read_excel(my_dataset)
+        else:
+            st.error("Unsupported file type. Please upload a CSV, Excel or HD5 file.")
+            st.stop()
+        parser = filter_dataframe(data)
+        st.session_state['parsed_responses'] = parser
+        st.dataframe(parser)
+        st.success(f"Successfully loaded and displayed data from {my_dataset.name}")
+        #h3_hex_id_df = pd.read_csv("keplergl/h3_data.csv")
+        st.session_state['data_loaded'] = True
+        # Load the base config
+        with open('military_config.kgl', 'r') as f:
+            base_config = json.load(f)
+
+        with open('uap_config.kgl', 'r') as f:
+            uap_config = json.load(f)
+
+        if parser.columns.str.contains('date').any():
+            # Get the date column name
+            date_column = parser.columns[parser.columns.str.contains('date')].values[0]
+
+            # Create a new filter
+            new_filter = {
+                "dataId": "uap_sightings",
+                "name": date_column
+            }
+
+            # Append the new filter to the existing filters
+            base_config['config']['visState']['filters'].append(new_filter)
+
+            # Update the map config
+            map_1.config = base_config
+
+        map_1.add_data(
+            data=parser, name="uap_sightings"
+            )
+        
+        # Find the latitude and longitude columns in the dataframe
+        lat_col, lon_col = find_lat_lon_columns(parser)
+
+        if lat_col and lon_col:
+            # Update the layer configurations
+            for layer in uap_config['config']['visState']['layers']:
+                if 'config' in layer and 'columns' in layer['config']:
+                    if 'lat' in layer['config']['columns']:
+                        layer['config']['columns']['lat'] = lat_col
+                    if 'lng' in layer['config']['columns']:
+                        layer['config']['columns']['lng'] = lon_col
+
+            # Now extend the base_config with the updated uap_config layers
+            base_config['config']['visState']['layers'].extend(uap_config['config']['visState']['layers'])
+            map_1.config = base_config
+        else:
+            base_config['config']['visState']['layers'].extend([layer for layer in uap_config['config']['visState']['layers']])
+            map_1.config = base_config
+        
+        keplergl_static(map_1, center_map=True)
+        st.session_state['map_generated'] = True
+    except Exception as e:
+        st.error(f"An error occurred while reading the file: {e}")
+else:
+    st.warning("Please upload a file to get started.")

military_config.kgl

@@ -0,0 +1,264 @@
+{
+  "version": "v1",
+  "config": {
+    "visState": {
+      "filters": [
+        {
+          "dataId": [
+            "nuclear_powerplants"
+          ],
+          "id": "c40zwvx0v",
+          "name": [
+            "primary_fuel"
+          ],
+          "type": "multiSelect",
+          "value": [
+            "Nuclear"
+          ],
+          "enlarged": false,
+          "plotType": "histogram",
+          "animationWindow": "free",
+          "yAxis": null,
+          "speed": 1
+        }
+      ],
+      "layers": [
+        {
+          "id": "k1xxw47",
+          "type": "icon",
+          "config": {
+            "dataId": "secret_bases",
+            "label": "Underground Bases",
+            "color": [
+              210,
+              0,
+              0
+            ],
+            "highlightColor": [
+              252,
+              242,
+              26,
+              255
+            ],
+            "columns": {
+              "lat": "latitude",
+              "lng": "longitude",
+              "icon": "icon"
+            },
+            "isVisible": true,
+            "visConfig": {
+              "radius": 35.9,
+              "fixedRadius": false,
+              "opacity": 0.8,
+              "colorRange": {
+                "name": "Global Warming",
+                "type": "sequential",
+                "category": "Uber",
+                "colors": [
+                  "#5A1846",
+                  "#900C3F",
+                  "#C70039",
+                  "#E3611C",
+                  "#F1920E",
+                  "#FFC300"
+                ]
+              },
+              "radiusRange": [
+                0,
+                50
+              ]
+            },
+            "hidden": false,
+            "textLabel": [
+              {
+                "field": null,
+                "color": [
+                  255,
+                  255,
+                  255
+                ],
+                "size": 18,
+                "offset": [
+                  0,
+                  0
+                ],
+                "anchor": "start",
+                "alignment": "center"
+              }
+            ]
+          },
+          "visualChannels": {
+            "colorField": null,
+            "colorScale": "quantile",
+            "sizeField": null,
+            "sizeScale": "linear"
+          }
+        },
+        {
+          "id": "i53syw",
+          "type": "icon",
+          "config": {
+            "dataId": "nuclear_powerplants",
+            "label": "Nuclear Facilities",
+            "color": [
+              253,
+              167,
+              0
+            ],
+            "highlightColor": [
+              252,
+              242,
+              26,
+              255
+            ],
+            "columns": {
+              "lat": "latitude",
+              "lng": "longitude",
+              "icon": "icon"
+            },
+            "isVisible": true,
+            "visConfig": {
+              "radius": 29.1,
+              "fixedRadius": false,
+              "opacity": 0.8,
+              "colorRange": {
+                "name": "Global Warming",
+                "type": "sequential",
+                "category": "Uber",
+                "colors": [
+                  "#5A1846",
+                  "#900C3F",
+                  "#C70039",
+                  "#E3611C",
+                  "#F1920E",
+                  "#FFC300"
+                ]
+              },
+              "radiusRange": [
+                0,
+                50
+              ]
+            },
+            "hidden": false,
+            "textLabel": [
+              {
+                "field": null,
+                "color": [
+                  255,
+                  255,
+                  255
+                ],
+                "size": 18,
+                "offset": [
+                  0,
+                  0
+                ],
+                "anchor": "start",
+                "alignment": "center"
+              }
+            ]
+          },
+          "visualChannels": {
+            "colorField": null,
+            "colorScale": "quantile",
+            "sizeField": null,
+            "sizeScale": "linear"
+          }
+        }
+      ],
+      "interactionConfig": {
+        "tooltip": {
+          "fieldsToShow": {
+            "qw5zqkhrp": [
+              {
+                "name": "0",
+                "format": null
+              },
+              {
+                "name": "country",
+                "format": null
+              },
+              {
+                "name": "country_long",
+                "format": null
+              },
+              {
+                "name": "name",
+                "format": null
+              },
+              {
+                "name": "gppd_idnr",
+                "format": null
+              }
+            ],
+            "hmakkovr9": [
+              {
+                "name": "0",
+                "format": null
+              },
+              {
+                "name": "id",
+                "format": null
+              },
+              {
+                "name": "name",
+                "format": null
+              },
+              {
+                "name": "icon",
+                "format": null
+              }
+            ]
+          },
+          "compareMode": false,
+          "compareType": "absolute",
+          "enabled": true
+        },
+        "brush": {
+          "size": 0.5,
+          "enabled": false
+        },
+        "geocoder": {
+          "enabled": false
+        },
+        "coordinate": {
+          "enabled": false
+        }
+      },
+      "layerBlending": "normal",
+      "splitMaps": [],
+      "animationConfig": {
+        "currentTime": null,
+        "speed": 1
+      }
+    },
+    "mapState": {
+      "bearing": 0,
+      "dragRotate": false,
+      "latitude": 34.502289455408366,
+      "longitude": -27.82946603675378,
+      "pitch": 0,
+      "zoom": 2.745704196646382,
+      "isSplit": false
+    },
+    "mapStyle": {
+      "styleType": "dark",
+      "topLayerGroups": {},
+      "visibleLayerGroups": {
+        "label": true,
+        "road": true,
+        "border": false,
+        "building": true,
+        "water": true,
+        "land": true,
+        "3d building": false
+      },
+      "threeDBuildingColor": [
+        9.665468314072013,
+        17.18305478057247,
+        31.1442867897876
+      ],
+      "mapStyles": {}
+    }
+  }
+}

navigation.py

@@ -0,0 +1,27 @@
+import streamlit as st
+from st_paywall import add_auth
+
+# add_auth(required=False)
+
+# st.write(st.session_state.email)
+# st.write(st.session_state.user_subscribed)
+
+# if "buttons" in st.session_state:
+#    st.session_state.buttons = st.session_state.buttons
+
+st.set_page_config(
+    page_title="UAP Analytics",
+    page_icon="🛸",
+    layout="wide",
+    initial_sidebar_state="expanded",
+)
+
+pg = st.navigation([
+            st.Page("rag_search.py", title="Smart-Search (Retrieval Augmented Generations)", icon="🔍"),
+            st.Page("parsing.py", title="UAP Feature Extraction (Shape, Speed, Color)", icon="📄"),
+            st.Page("analyzing.py", title="Statistical Analysis (UMAP+HDBSCAN, XGBoost, V-Cramer)", icon="🧠"),
+            st.Page("magnetic.py", title="Magnetic Anomaly Detection (InterMagnet Stations)", icon="🧲"),
+            st.Page("map.py", title="Interactive Map", icon="🗺️"),
+        ])
+
+pg.run()

parsing.py

@@ -0,0 +1,678 @@
+import streamlit as st
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import seaborn as sns
+from uap_analyzer import UAPParser, UAPAnalyzer, UAPVisualizer
+# import ChartGen
+# from ChartGen import ChartGPT
+from Levenshtein import distance
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import confusion_matrix
+from stqdm import stqdm
+stqdm.pandas()
+import streamlit.components.v1 as components
+from dateutil import parser
+from sentence_transformers import SentenceTransformer
+import torch
+import squarify
+import matplotlib.colors as mcolors
+import textwrap
+import datamapplot
+import openai
+from openai import OpenAI
+import os
+import json
+# this is a test comment
+import plotly.graph_objects as go
+
+st.set_option('deprecation.showPyplotGlobalUse', False)
+
+from pandas.api.types import (
+    is_categorical_dtype,
+    is_datetime64_any_dtype,
+    is_numeric_dtype,
+    is_object_dtype,
+)
+
+
+
+def load_data(file_path, key='df'):
+    return pd.read_hdf(file_path, key=key)
+
+
+def gemini_query(question, selected_data, gemini_key):
+
+    if question == "":
+        question = "Summarize the following data in relevant bullet points"
+
+    import pathlib
+    import textwrap
+
+    import google.generativeai as genai
+
+    from IPython.display import display
+    from IPython.display import Markdown
+
+
+    def to_markdown(text):
+        text = text.replace('•', '  *')
+        return Markdown(textwrap.indent(text, '> ', predicate=lambda _: True))
+    
+    # selected_data is a list
+    # remove empty
+
+    filtered = [str(x) for x in selected_data if str(x) != '' and x is not None]
+    # make a string
+    context = '\n'.join(filtered)
+
+    genai.configure(api_key=gemini_key)
+    query_model = genai.GenerativeModel('models/gemini-1.5-pro-latest')
+    response = query_model.generate_content([f"{question}\n Answer based on this context: {context}\n\n"])
+    return(response.text)
+
+def plot_treemap(df, column, top_n=32):
+        # Get the value counts and the top N labels
+        value_counts = df[column].value_counts()
+        top_labels = value_counts.iloc[:top_n].index
+        
+        # Use np.where to replace all values not in the top N with 'Other'
+        revised_column = f'{column}_revised'
+        df[revised_column] = np.where(df[column].isin(top_labels), df[column], 'Other')
+
+        # Get the value counts including the 'Other' category
+        sizes = df[revised_column].value_counts().values
+        labels = df[revised_column].value_counts().index
+
+        # Get a gradient of colors
+        # colors = list(mcolors.TABLEAU_COLORS.values())
+
+        n_colors = len(sizes)
+        colors = plt.cm.Oranges(np.linspace(0.3, 0.9, n_colors))[::-1]
+
+
+        # Get % of each category
+        percents = sizes / sizes.sum()
+
+        # Prepare labels with percentages
+        labels = [f'{label}\n {percent:.1%}' for label, percent in zip(labels, percents)]
+
+        fig, ax = plt.subplots(figsize=(20, 12))
+
+        # Plot the treemap
+        squarify.plot(sizes=sizes, label=labels, alpha=0.7, pad=True, color=colors, text_kwargs={'fontsize': 10})
+
+        ax = plt.gca()
+        # Iterate over text elements and rectangles (patches) in the axes for color adjustment
+        for text, rect in zip(ax.texts, ax.patches):
+            background_color = rect.get_facecolor()
+            r, g, b, _ = mcolors.to_rgba(background_color)
+            brightness = np.average([r, g, b])
+            text.set_color('white' if brightness < 0.5 else 'black')
+
+            # Adjust font size based on rectangle's area and wrap long text
+  
+
+st.set_option('deprecation.showPyplotGlobalUse', False)
+
+from pandas.api.types import (
+    is_categorical_dtype,
+    is_datetime64_any_dtype,
+    is_numeric_dtype,
+    is_object_dtype,
+)
+
+
+class CachedUAPParser(UAPParser):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if 'parsed_responses' not in st.session_state:
+            st.session_state['parsed_responses'] = {}
+
+    def parse_responses(self):
+        parsed_responses = {}
+        not_parsed = 0
+        try:
+            for k, v in self.responses.items():
+                try:
+                    parsed_responses[k] = json.loads(v)
+                except:
+                    try:
+                        parsed_responses[k] = json.loads(v.replace("'", '"'))
+                    except:
+                        not_parsed += 1
+
+            # Update the cached responses
+            st.session_state['parsed_responses'] = parsed_responses
+        except Exception as e:
+            st.error(f"Error parsing responses: {e}")
+ 
+        st.write(f"Number of unparsed responses: {not_parsed}")
+        st.write(f"Number of parsed responses: {len(parsed_responses)}")
+        return st.session_state['parsed_responses']
+
+    def responses_to_df(self, col, parsed_responses):
+        try:
+            parsed_df = pd.DataFrame(parsed_responses).T
+            if col is not None:
+                parsed_df2 = pd.json_normalize(parsed_df[col])
+                parsed_df2.index = parsed_df.index
+            else:
+                parsed_df2 = pd.json_normalize(parsed_df)
+                parsed_df2.index = parsed_df.index
+            
+            # Convert problematic columns to string
+            for column in parsed_df2.columns:
+                if parsed_df2[column].dtype == 'object':
+                    parsed_df2[column] = parsed_df2[column].astype(str)
+            
+            return parsed_df2
+        except Exception as e:
+            st.error(f"Error converting responses to DataFrame: {e}")
+            return pd.DataFrame()  # Return an empty DataFrame if conversion fails
+
+
+def load_data(file_path, key='df'):
+    return pd.read_hdf(file_path, key=key)
+
+
+def gemini_query(question, selected_data, gemini_key):
+
+    if question == "":
+        question = "Summarize the following data in relevant bullet points"
+
+    import pathlib
+    import textwrap
+
+    import google.generativeai as genai
+
+    from IPython.display import display
+    from IPython.display import Markdown
+
+
+    def to_markdown(text):
+        text = text.replace('•', '  *')
+        return Markdown(textwrap.indent(text, '> ', predicate=lambda _: True))
+    
+    # selected_data is a list
+    # remove empty
+
+    filtered = [str(x) for x in selected_data if str(x) != '' and x is not None]
+    # make a string
+    context = '\n'.join(filtered)
+
+    genai.configure(api_key=gemini_key)
+    query_model = genai.GenerativeModel('models/gemini-1.5-pro-latest')
+    response = query_model.generate_content([f"{question}\n Answer based on this context: {context}\n\n"])
+    return(response.text)
+
+
+def plot_hist(df, column, bins=10, kde=True):
+        fig, ax = plt.subplots(figsize=(12, 6))
+        sns.histplot(data=df, x=column, kde=True, bins=bins,color='orange')
+        # set the ticks and frame in orange
+        ax.spines['bottom'].set_color('orange')
+        ax.spines['top'].set_color('orange')
+        ax.spines['right'].set_color('orange')
+        ax.spines['left'].set_color('orange')
+        ax.xaxis.label.set_color('orange')
+        ax.yaxis.label.set_color('orange')
+        ax.tick_params(axis='x', colors='orange')
+        ax.tick_params(axis='y', colors='orange')
+        ax.title.set_color('orange')
+
+        # Set transparent background
+        fig.patch.set_alpha(0)
+        ax.patch.set_alpha(0)
+        return fig
+
+
+
+def is_api_key_valid(api_key, model='gpt-3.5-turbo'):
+    try:
+        os.environ['OPENAI_API_KEY'] = api_key
+        client = OpenAI()
+        response = client.chat.completions.create(
+            model=model,
+            messages=[{"role": "user", "content": 'Say Hello World!'}])
+        text = response.choices[0].message.content
+        if len(text) >= 0:
+            return True
+    except Exception as e:
+       st.error(f'Error with the API key :{e}')
+       return False
+
+def download_json(data):
+    json_str = json.dumps(data, indent=2)
+    return json_str
+
+
+def convert_cached_data_to_df(parser):
+    if 'parsed_responses' in st.session_state:
+        #parser = CachedUAPParser(api_key=API_KEY, model='gpt-3.5-turbo-0125')
+        try:
+            responses_df = parser.responses_to_df('sightingDetails', st.session_state['parsed_responses'])
+        except Exception as e:
+            st.warning(f"Error parsing with 'sightingDetails': {e}")
+            responses_df = parser.responses_to_df(None, st.session_state['parsed_responses'])
+        if not responses_df.empty:
+            st.dataframe(responses_df)
+            st.session_state['parsed_responses_df'] = responses_df.copy()
+            st.success("Successfully converted cached data to DataFrame.")
+        else:
+            st.error("Failed to create DataFrame from cached responses.")
+    else:
+        st.warning("No cached data available. Please parse the dataset first.")
+
+def plot_line(df, x_column, y_columns, figsize=(12, 10), color='orange', title=None, rolling_mean_value=2):
+    import matplotlib.cm as cm
+    # Sort the dataframe by the date column
+    df = df.sort_values(by=x_column)
+
+    # Calculate rolling mean for each y_column
+    if rolling_mean_value:
+        df[y_columns] = df[y_columns].rolling(len(df) // rolling_mean_value).mean()
+
+    # Create the plot
+    fig, ax = plt.subplots(figsize=figsize)
+
+    colors = cm.Oranges(np.linspace(0.2, 1, len(y_columns)))
+
+    # Plot each y_column as a separate line with a different color
+    for i, y_column in enumerate(y_columns):
+        df.plot(x=x_column, y=y_column, ax=ax, color=colors[i], label=y_column, linewidth=.5)
+
+    # Rotate x-axis labels
+    ax.set_xticklabels(ax.get_xticklabels(), rotation=30, ha='right')
+
+    # Format x_column as date if it is
+    if np.issubdtype(df[x_column].dtype, np.datetime64) or np.issubdtype(df[x_column].dtype, np.timedelta64):
+        df[x_column] = pd.to_datetime(df[x_column]).dt.date
+
+    # Set title, labels, and legend
+    ax.set_title(title or f'{", ".join(y_columns)} over {x_column}', color=color, fontweight='bold')
+    ax.set_xlabel(x_column, color=color)
+    ax.set_ylabel(', '.join(y_columns), color=color)
+    ax.spines['bottom'].set_color('orange')
+    ax.spines['top'].set_color('orange')
+    ax.spines['right'].set_color('orange')
+    ax.spines['left'].set_color('orange')
+    ax.xaxis.label.set_color('orange')
+    ax.yaxis.label.set_color('orange')
+    ax.tick_params(axis='x', colors='orange')
+    ax.tick_params(axis='y', colors='orange')
+    ax.title.set_color('orange')
+
+    ax.legend(loc='upper right', bbox_to_anchor=(1, 1), facecolor='black', framealpha=.4, labelcolor='orange', edgecolor='orange')
+
+    # Remove background
+    fig.patch.set_alpha(0)
+    ax.patch.set_alpha(0)
+
+    return fig
+
+def plot_bar(df, x_column, y_column, figsize=(12, 10), color='orange', title=None):
+    fig, ax = plt.subplots(figsize=figsize)
+
+    sns.barplot(data=df, x=x_column, y=y_column, color=color, ax=ax)
+
+    ax.set_title(title if title else f'{y_column} by {x_column}', color=color, fontweight='bold')
+    ax.set_xlabel(x_column, color=color)
+    ax.set_ylabel(y_column, color=color)
+
+    ax.tick_params(axis='x', colors=color)
+    ax.tick_params(axis='y', colors=color)
+
+    # Remove background
+    fig.patch.set_alpha(0)
+    ax.patch.set_alpha(0)
+    ax.spines['bottom'].set_color('orange')
+    ax.spines['top'].set_color('orange')
+    ax.spines['right'].set_color('orange')
+    ax.spines['left'].set_color('orange')
+    ax.xaxis.label.set_color('orange')
+    ax.yaxis.label.set_color('orange')
+    ax.tick_params(axis='x', colors='orange')
+    ax.tick_params(axis='y', colors='orange')
+    ax.title.set_color('orange')
+    ax.legend(loc='upper right', bbox_to_anchor=(1, 1), facecolor='black', framealpha=.4, labelcolor='orange', edgecolor='orange')
+
+    return fig
+
+def plot_grouped_bar(df, x_columns, y_column, figsize=(12, 10), colors=None, title=None):
+    fig, ax = plt.subplots(figsize=figsize)
+
+    width = 0.8 / len(x_columns)  # the width of the bars
+    x = np.arange(len(df))  # the label locations
+
+    for i, x_column in enumerate(x_columns):
+        sns.barplot(data=df, x=x, y=y_column, color=colors[i] if colors else None, ax=ax, width=width, label=x_column)
+        x += width  # add the width of the bar to the x position for the next bar
+
+    ax.set_title(title if title else f'{y_column} by {", ".join(x_columns)}', color='orange', fontweight='bold')
+    ax.set_xlabel('Groups', color='orange')
+    ax.set_ylabel(y_column, color='orange')
+
+    ax.set_xticks(x - width * len(x_columns) / 2)
+    ax.set_xticklabels(df.index)
+
+    ax.tick_params(axis='x', colors='orange')
+    ax.tick_params(axis='y', colors='orange')
+
+    # Remove background
+    fig.patch.set_alpha(0)
+    ax.patch.set_alpha(0)
+    ax.spines['bottom'].set_color('orange')
+    ax.spines['top'].set_color('orange')
+    ax.spines['right'].set_color('orange')
+    ax.spines['left'].set_color('orange')
+    ax.xaxis.label.set_color('orange')
+    ax.yaxis.label.set_color('orange')
+    ax.title.set_color('orange')
+    ax.legend(loc='upper right', bbox_to_anchor=(1, 1), facecolor='black', framealpha=.4, labelcolor='orange', edgecolor='orange')
+
+    return fig
+
+@st.cache_data
+def convert_df(df):
+    # IMPORTANT: Cache the conversion to prevent computation on every rerun
+    try:
+        csv = df.to_csv().encode("utf-8")
+    except:
+        csv = df.to_csv().encode("utf-8-sig")
+    return csv
+
+
+def filter_dataframe(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Adds a UI on top of a dataframe to let viewers filter columns
+
+    Args:
+        df (pd.DataFrame): Original dataframe
+
+    Returns:
+        pd.DataFrame: Filtered dataframe
+    """
+
+    title_font = "Arial"
+    body_font = "Arial"
+    title_size = 32
+    colors = ["red", "green", "blue"]
+    interpretation = False
+    extract_docx = False
+    title = "My Chart"
+    regex = ".*"
+    img_path = 'default_image.png'
+
+
+    #try:
+    #    modify = st.checkbox("Add filters on raw data")
+    #except:
+    #    try:
+    #        modify = st.checkbox("Add filters on processed data")
+    #    except:
+    #        try:
+    #            modify = st.checkbox("Add filters on parsed data")
+    #        except:
+    #            pass
+
+    #if not modify:
+    #    return df
+
+    df_ = df.copy()
+    # Try to convert datetimes into a standard format (datetime, no timezone)
+
+#modification_container = st.container()
+
+#with modification_container:
+    to_filter_columns = st.multiselect("Filter dataframe on", df_.columns)
+
+    date_column = None
+    filtered_columns = []
+
+    for column in to_filter_columns:
+        left, right = st.columns((1, 20))
+        # Treat columns with < 200 unique values as categorical if not date or numeric
+        if is_categorical_dtype(df_[column]) or (df_[column].nunique() < 120 and not is_datetime64_any_dtype(df_[column]) and not is_numeric_dtype(df_[column])):
+            user_cat_input = right.multiselect(
+                f"Values for {column}",
+                df_[column].value_counts().index.tolist(),
+                default=list(df_[column].value_counts().index)
+            )
+            df_ = df_[df_[column].isin(user_cat_input)]
+            filtered_columns.append(column)
+
+            with st.status(f"Category Distribution: {column}", expanded=False) as stat:
+                st.pyplot(plot_treemap(df_, column))
+
+        elif is_numeric_dtype(df_[column]):
+            _min = float(df_[column].min())
+            _max = float(df_[column].max())
+            step = (_max - _min) / 100
+            user_num_input = right.slider(
+                f"Values for {column}",
+                min_value=_min,
+                max_value=_max,
+                value=(_min, _max),
+                step=step,
+            )
+            df_ = df_[df_[column].between(*user_num_input)]
+            filtered_columns.append(column)
+
+            # Chart_GPT = ChartGPT(df_, title_font, body_font, title_size,
+            #      colors, interpretation, extract_docx, img_path)
+
+            with st.status(f"Numerical Distribution: {column}", expanded=False) as stat_:
+                st.pyplot(plot_hist(df_, column, bins=int(round(len(df_[column].unique())-1)/2)))
+
+        elif is_object_dtype(df_[column]):
+            try:
+                df_[column] = pd.to_datetime(df_[column], infer_datetime_format=True, errors='coerce')
+            except Exception:
+                try:
+                    df_[column] = df_[column].apply(parser.parse)
+                except Exception:
+                    pass
+
+            if is_datetime64_any_dtype(df_[column]):
+                df_[column] = df_[column].dt.tz_localize(None)
+                min_date = df_[column].min().date()
+                max_date = df_[column].max().date()
+                user_date_input = right.date_input(
+                    f"Values for {column}",
+                    value=(min_date, max_date),
+                    min_value=min_date,
+                    max_value=max_date,
+                )
+                # if len(user_date_input) == 2:
+                #     start_date, end_date = user_date_input
+                #     df_ = df_.loc[df_[column].dt.date.between(start_date, end_date)]
+                if len(user_date_input) == 2:
+                    user_date_input = tuple(map(pd.to_datetime, user_date_input))
+                    start_date, end_date = user_date_input
+                    df_ = df_.loc[df_[column].between(start_date, end_date)]
+
+                date_column = column
+
+                if date_column and filtered_columns:
+                    numeric_columns = [col for col in filtered_columns if is_numeric_dtype(df_[col])]
+                    if numeric_columns:
+                        fig = plot_line(df_, date_column, numeric_columns)
+                        #st.pyplot(fig)
+                    # now to deal with categorical columns
+                    categorical_columns = [col for col in filtered_columns if is_categorical_dtype(df_[col])]
+                    if categorical_columns:
+                        fig2 = plot_bar(df_, date_column, categorical_columns[0])
+                        #st.pyplot(fig2)
+                    with st.status(f"Date Distribution: {column}", expanded=False) as stat:
+                        try:
+                            st.pyplot(fig)
+                        except Exception as e:
+                            st.error(f"Error plotting line chart: {e}")
+                            pass
+                        try:
+                            st.pyplot(fig2)
+                        except Exception as e:
+                            st.error(f"Error plotting bar chart: {e}")
+
+
+        else:
+            user_text_input = right.text_input(
+                f"Substring or regex in {column}",
+            )
+            if user_text_input:
+                df_ = df_[df_[column].astype(str).str.contains(user_text_input)]
+    # write len of df after filtering with % of original
+    st.write(f"{len(df_)} rows ({len(df_) / len(df) * 100:.2f}%)")
+    return df_
+
+
+from config import API_KEY, GEMINI_KEY, FORMAT_LONG
+
+with torch.no_grad():
+    torch.cuda.empty_cache()
+
+#st.set_page_config(
+#    page_title="UAP ANALYSIS",
+#    page_icon=":alien:",
+#    layout="wide",
+#    initial_sidebar_state="expanded",
+#)
+
+st.title('UAP Feature Extraction')
+
+# Initialize session state
+if 'analyzers' not in st.session_state:
+    st.session_state['analyzers'] = []
+if 'col_names' not in st.session_state:
+    st.session_state['col_names'] = []
+if 'clusters' not in st.session_state:
+    st.session_state['clusters'] = {}
+if 'new_data' not in st.session_state:
+    st.session_state['new_data'] = pd.DataFrame()
+if 'dataset' not in st.session_state:
+    st.session_state['dataset'] = pd.DataFrame()
+if 'data_processed' not in st.session_state:
+    st.session_state['data_processed'] = False
+if 'stage' not in st.session_state:
+    st.session_state['stage'] = 0
+if 'filtered_data' not in st.session_state:
+    st.session_state['filtered_data'] = None
+if 'gemini_answer' not in st.session_state:
+    st.session_state['gemini_answer'] = None
+if 'parsed_responses' not in st.session_state:
+    st.session_state['parsed_responses'] = None
+if 'parsed_responses_df' not in st.session_state:
+    st.session_state['parsed_responses_df'] = None
+if 'json_format' not in st.session_state:
+    st.session_state['json_format'] = None
+if 'api_key_valid' not in st.session_state:
+    st.session_state['api_key_valid'] = False
+if 'previous_api_key' not in st.session_state:
+    st.session_state['previous_api_key'] = None
+
+    
+# Unparsed data
+#unparsed_tickbox = st.checkbox('Data Parsing')
+#if unparsed_tickbox:
+unparsed = st.file_uploader("Upload Raw DataFrame", type=["csv", "xlsx"])
+if unparsed is not None:
+    try:
+        data = pd.read_csv(unparsed) if unparsed.type == "text/csv" else pd.read_excel(unparsed)
+        filtered_data = filter_dataframe(data)
+        st.dataframe(filtered_data)
+    except Exception as e:
+        st.error(f"An error occurred while reading the file: {e}")
+    
+    modify_json = st.checkbox('Custom JSON')
+    API_KEY = st.text_input('OpenAI API Key', API_KEY, type='password', help="Enter your OpenAI API key")       
+        
+    
+    
+    if modify_json:
+        FORMAT_LONG = st.text_area('Custom JSON', FORMAT_LONG, height=500)
+        st.download_button("Save Format", FORMAT_LONG)
+    try:
+        json.loads(FORMAT_LONG)
+        st.session_state['json_format'] = True
+    except json.JSONDecodeError as e:
+        st.error(f"Invalid JSON format: {str(e)}")
+        st.session_state['json_format'] = False
+        st.stop()  # Stop execution if JSON is invalid
+
+    # If the DataFrame is successfully created, allow the user to select a column
+    col_unparsed = st.selectbox("Select column corresponding to text", data.columns)
+    
+        
+    if st.button("Parse Dataset") and st.session_state['json_format']:
+        if API_KEY:
+            # Only validate if the API key has changed
+            if API_KEY != st.session_state['previous_api_key']:
+                if is_api_key_valid(API_KEY):
+                    st.session_state['api_key_valid'] = True
+                    st.session_state['previous_api_key'] = API_KEY
+                    st.success("API key is valid!")
+                else:
+                    st.session_state['api_key_valid'] = False
+                    st.error("Invalid API key. Please check and try again.")
+            elif st.session_state['api_key_valid']:
+                st.success("API key is valid!")
+        if not API_KEY:# or not st.session_state['api_key_valid']:
+            st.warning("Please enter your API key to proceed.")
+            st.stop()
+        selected_column_data = filtered_data[col_unparsed].tolist()
+        st.session_state.result = selected_column_data
+        with st.status("Parsing...", expanded=True) as stat:
+            try:
+                st.write("Parsing descriptions...")
+                parser = CachedUAPParser(api_key=API_KEY, model='gpt-3.5-turbo-0125', col=st.session_state.result)
+                descriptions = st.session_state.result
+                format_long = FORMAT_LONG
+                parser.process_descriptions(descriptions, format_long)
+                st.session_state['parsed_responses'] = parser.parse_responses()
+                try:
+                    responses_df = parser.responses_to_df('sightingDetails', st.session_state['parsed_responses'])
+                except Exception as e:
+                    st.warning(f"Error parsing with 'sightingDetails': {e}")
+                    responses_df = parser.responses_to_df(None, st.session_state['parsed_responses'])
+                
+                if not responses_df.empty:
+                    st.dataframe(responses_df)
+                    st.session_state['parsed_responses_df'] = responses_df.copy()
+                    stat.update(label="Parsing complete", state="complete", expanded=False)
+                else:
+                    st.error("Failed to create DataFrame from parsed responses.")
+            except Exception as e:
+                st.error(f"An error occurred during parsing: {str(e)}")
+
+    # Add download button for parsed data
+    if st.session_state['parsed_responses'] is not None:
+        json_str = download_json(st.session_state['parsed_responses'])
+        st.download_button(
+            label="Download Parsed Data as JSON",
+            data=json_str,
+            file_name="parsed_responses.json",
+            mime="application/json"
+        )
+        # Add button to convert cached data to DataFrame
+        if st.button("Convert Cached Data to DataFrame"):
+            convert_cached_data_to_df(st.session_state['parsed_responses'])
+            
+    if st.session_state['parsed_responses_df'] is not None:
+        st.download_button(
+        label="Save CSV",
+        data=convert_df(st.session_state['parsed_responses_df']),
+        file_name="uap_data.csv",
+        mime="text/csv",
+        )
+
+    
+        
+
+
+
+#except Exception as e:
+#    stat.update(label=f"Parsing failed: {e}", state="error")
+# st.write("Parsing descriptions...")
+# st.update_status("Parsing descriptions...")
+ 

rag_search.py

@@ -0,0 +1,438 @@
+
+import streamlit as st
+import pandas as pd
+import cohere
+import streamlit as st
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import seaborn as sns
+from uap_analyzer import UAPParser, UAPAnalyzer, UAPVisualizer
+# import ChartGen
+# from ChartGen import ChartGPT
+from Levenshtein import distance
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import confusion_matrix
+from stqdm import stqdm
+stqdm.pandas()
+import streamlit.components.v1 as components
+from dateutil import parser
+from sentence_transformers import SentenceTransformer
+import torch
+import squarify
+import matplotlib.colors as mcolors
+import textwrap
+import datamapplot
+import json
+
+st.set_option('deprecation.showPyplotGlobalUse', False)
+
+from pandas.api.types import (
+    is_categorical_dtype,
+    is_datetime64_any_dtype,
+    is_numeric_dtype,
+    is_object_dtype,
+)
+
+
+def plot_treemap(df, column, top_n=32):
+        # Get the value counts and the top N labels
+        value_counts = df[column].value_counts()
+        top_labels = value_counts.iloc[:top_n].index
+        
+        # Use np.where to replace all values not in the top N with 'Other'
+        revised_column = f'{column}_revised'
+        df[revised_column] = np.where(df[column].isin(top_labels), df[column], 'Other')
+
+        # Get the value counts including the 'Other' category
+        sizes = df[revised_column].value_counts().values
+        labels = df[revised_column].value_counts().index
+
+        # Get a gradient of colors
+        # colors = list(mcolors.TABLEAU_COLORS.values())
+
+        n_colors = len(sizes)
+        colors = plt.cm.Oranges(np.linspace(0.3, 0.9, n_colors))[::-1]
+
+
+        # Get % of each category
+        percents = sizes / sizes.sum()
+
+        # Prepare labels with percentages
+        labels = [f'{label}\n {percent:.1%}' for label, percent in zip(labels, percents)]
+
+        fig, ax = plt.subplots(figsize=(20, 12))
+
+        # Plot the treemap
+        squarify.plot(sizes=sizes, label=labels, alpha=0.7, pad=True, color=colors, text_kwargs={'fontsize': 10})
+
+        ax = plt.gca()
+        # Iterate over text elements and rectangles (patches) in the axes for color adjustment
+        for text, rect in zip(ax.texts, ax.patches):
+            background_color = rect.get_facecolor()
+            r, g, b, _ = mcolors.to_rgba(background_color)
+            brightness = np.average([r, g, b])
+            text.set_color('white' if brightness < 0.5 else 'black')
+
+            # Adjust font size based on rectangle's area and wrap long text
+            coef = 0.8
+            font_size = np.sqrt(rect.get_width() * rect.get_height()) * coef
+            text.set_fontsize(font_size)
+            wrapped_text = textwrap.fill(text.get_text(), width=20)
+            text.set_text(wrapped_text)
+
+        plt.axis('off')
+        plt.gca().invert_yaxis()
+        plt.gcf().set_size_inches(20, 12)
+
+        fig.patch.set_alpha(0)
+
+        ax.patch.set_alpha(0)
+        return fig
+
+def plot_hist(df, column, bins=10, kde=True):
+        fig, ax = plt.subplots(figsize=(12, 6))
+        sns.histplot(data=df, x=column, kde=True, bins=bins,color='orange')
+        # set the ticks and frame in orange
+        ax.spines['bottom'].set_color('orange')
+        ax.spines['top'].set_color('orange')
+        ax.spines['right'].set_color('orange')
+        ax.spines['left'].set_color('orange')
+        ax.xaxis.label.set_color('orange')
+        ax.yaxis.label.set_color('orange')
+        ax.tick_params(axis='x', colors='orange')
+        ax.tick_params(axis='y', colors='orange')
+        ax.title.set_color('orange')
+
+        # Set transparent background
+        fig.patch.set_alpha(0)
+        ax.patch.set_alpha(0)
+        return fig
+
+
+def plot_line(df, x_column, y_columns, figsize=(12, 10), color='orange', title=None, rolling_mean_value=2):
+    import matplotlib.cm as cm
+    # Sort the dataframe by the date column
+    df = df.sort_values(by=x_column)
+
+    # Calculate rolling mean for each y_column
+    if rolling_mean_value:
+        df[y_columns] = df[y_columns].rolling(len(df) // rolling_mean_value).mean()
+
+    # Create the plot
+    fig, ax = plt.subplots(figsize=figsize)
+
+    colors = cm.Oranges(np.linspace(0.2, 1, len(y_columns)))
+
+    # Plot each y_column as a separate line with a different color
+    for i, y_column in enumerate(y_columns):
+        df.plot(x=x_column, y=y_column, ax=ax, color=colors[i], label=y_column, linewidth=.5)
+
+    # Rotate x-axis labels
+    ax.set_xticklabels(ax.get_xticklabels(), rotation=30, ha='right')
+
+    # Format x_column as date if it is
+    if np.issubdtype(df[x_column].dtype, np.datetime64) or np.issubdtype(df[x_column].dtype, np.timedelta64):
+        df[x_column] = pd.to_datetime(df[x_column]).dt.date
+
+    # Set title, labels, and legend
+    ax.set_title(title or f'{", ".join(y_columns)} over {x_column}', color=color, fontweight='bold')
+    ax.set_xlabel(x_column, color=color)
+    ax.set_ylabel(', '.join(y_columns), color=color)
+    ax.spines['bottom'].set_color('orange')
+    ax.spines['top'].set_color('orange')
+    ax.spines['right'].set_color('orange')
+    ax.spines['left'].set_color('orange')
+    ax.xaxis.label.set_color('orange')
+    ax.yaxis.label.set_color('orange')
+    ax.tick_params(axis='x', colors='orange')
+    ax.tick_params(axis='y', colors='orange')
+    ax.title.set_color('orange')
+
+    ax.legend(loc='upper right', bbox_to_anchor=(1, 1), facecolor='black', framealpha=.4, labelcolor='orange', edgecolor='orange')
+
+    # Remove background
+    fig.patch.set_alpha(0)
+    ax.patch.set_alpha(0)
+
+    return fig
+
+def plot_bar(df, x_column, y_column, figsize=(12, 10), color='orange', title=None):
+    fig, ax = plt.subplots(figsize=figsize)
+
+    sns.barplot(data=df, x=x_column, y=y_column, color=color, ax=ax)
+
+    ax.set_title(title if title else f'{y_column} by {x_column}', color=color, fontweight='bold')
+    ax.set_xlabel(x_column, color=color)
+    ax.set_ylabel(y_column, color=color)
+
+    ax.tick_params(axis='x', colors=color)
+    ax.tick_params(axis='y', colors=color)
+
+    # Remove background
+    fig.patch.set_alpha(0)
+    ax.patch.set_alpha(0)
+    ax.spines['bottom'].set_color('orange')
+    ax.spines['top'].set_color('orange')
+    ax.spines['right'].set_color('orange')
+    ax.spines['left'].set_color('orange')
+    ax.xaxis.label.set_color('orange')
+    ax.yaxis.label.set_color('orange')
+    ax.tick_params(axis='x', colors='orange')
+    ax.tick_params(axis='y', colors='orange')
+    ax.title.set_color('orange')
+    ax.legend(loc='upper right', bbox_to_anchor=(1, 1), facecolor='black', framealpha=.4, labelcolor='orange', edgecolor='orange')
+
+    return fig
+
+def plot_grouped_bar(df, x_columns, y_column, figsize=(12, 10), colors=None, title=None):
+    fig, ax = plt.subplots(figsize=figsize)
+
+    width = 0.8 / len(x_columns)  # the width of the bars
+    x = np.arange(len(df))  # the label locations
+
+    for i, x_column in enumerate(x_columns):
+        sns.barplot(data=df, x=x, y=y_column, color=colors[i] if colors else None, ax=ax, width=width, label=x_column)
+        x += width  # add the width of the bar to the x position for the next bar
+
+    ax.set_title(title if title else f'{y_column} by {", ".join(x_columns)}', color='orange', fontweight='bold')
+    ax.set_xlabel('Groups', color='orange')
+    ax.set_ylabel(y_column, color='orange')
+
+    ax.set_xticks(x - width * len(x_columns) / 2)
+    ax.set_xticklabels(df.index)
+
+    ax.tick_params(axis='x', colors='orange')
+    ax.tick_params(axis='y', colors='orange')
+
+    # Remove background
+    fig.patch.set_alpha(0)
+    ax.patch.set_alpha(0)
+    ax.spines['bottom'].set_color('orange')
+    ax.spines['top'].set_color('orange')
+    ax.spines['right'].set_color('orange')
+    ax.spines['left'].set_color('orange')
+    ax.xaxis.label.set_color('orange')
+    ax.yaxis.label.set_color('orange')
+    ax.title.set_color('orange')
+    ax.legend(loc='upper right', bbox_to_anchor=(1, 1), facecolor='black', framealpha=.4, labelcolor='orange', edgecolor='orange')
+
+    return fig
+
+
+def filter_dataframe(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Adds a UI on top of a dataframe to let viewers filter columns
+
+    Args:
+        df (pd.DataFrame): Original dataframe
+
+    Returns:
+        pd.DataFrame: Filtered dataframe
+    """
+
+    title_font = "Arial"
+    body_font = "Arial"
+    title_size = 32
+    colors = ["red", "green", "blue"]
+    interpretation = False
+    extract_docx = False
+    title = "My Chart"
+    regex = ".*"
+    img_path = 'default_image.png'
+
+    df_ = df.copy()
+
+#modification_container = st.container()
+
+#with modification_container:
+    to_filter_columns = st.multiselect("Filter dataframe on", df_.columns)
+
+    date_column = None
+    filtered_columns = []
+
+    for column in to_filter_columns:
+        left, right = st.columns((1, 20))
+        # Treat columns with < 200 unique values as categorical if not date or numeric
+        if is_categorical_dtype(df_[column]) or (df_[column].nunique() < 120 and not is_datetime64_any_dtype(df_[column]) and not is_numeric_dtype(df_[column])):
+            user_cat_input = right.multiselect(
+                f"Values for {column}",
+                df_[column].value_counts().index.tolist(),
+                default=list(df_[column].value_counts().index)
+            )
+            df_ = df_[df_[column].isin(user_cat_input)]
+            filtered_columns.append(column)
+
+            with st.status(f"Category Distribution: {column}", expanded=False) as stat:
+                st.pyplot(plot_treemap(df_, column))
+
+        elif is_numeric_dtype(df_[column]):
+            _min = float(df_[column].min())
+            _max = float(df_[column].max())
+            step = (_max - _min) / 100
+            user_num_input = right.slider(
+                f"Values for {column}",
+                min_value=_min,
+                max_value=_max,
+                value=(_min, _max),
+                step=step,
+            )
+            df_ = df_[df_[column].between(*user_num_input)]
+            filtered_columns.append(column)
+
+            # Chart_GPT = ChartGPT(df_, title_font, body_font, title_size,
+            #      colors, interpretation, extract_docx, img_path)
+
+            with st.status(f"Numerical Distribution: {column}", expanded=False) as stat_:
+                st.pyplot(plot_hist(df_, column, bins=int(round(len(df_[column].unique())-1)/2)))
+
+        elif is_object_dtype(df_[column]):
+            try:
+                df_[column] = pd.to_datetime(df_[column], infer_datetime_format=True, errors='coerce')
+            except Exception:
+                try:
+                    df_[column] = df_[column].apply(parser.parse)
+                except Exception:
+                    pass
+
+            if is_datetime64_any_dtype(df_[column]):
+                df_[column] = df_[column].dt.tz_localize(None)
+                min_date = df_[column].min().date()
+                max_date = df_[column].max().date()
+                user_date_input = right.date_input(
+                    f"Values for {column}",
+                    value=(min_date, max_date),
+                    min_value=min_date,
+                    max_value=max_date,
+                )
+                # if len(user_date_input) == 2:
+                #     start_date, end_date = user_date_input
+                #     df_ = df_.loc[df_[column].dt.date.between(start_date, end_date)]
+                if len(user_date_input) == 2:
+                    user_date_input = tuple(map(pd.to_datetime, user_date_input))
+                    start_date, end_date = user_date_input
+                    df_ = df_.loc[df_[column].between(start_date, end_date)]
+
+                date_column = column
+
+                if date_column and filtered_columns:
+                    numeric_columns = [col for col in filtered_columns if is_numeric_dtype(df_[col])]
+                    if numeric_columns:
+                        fig = plot_line(df_, date_column, numeric_columns)
+                        with st.status(f"Date Numerical Distributions: {column}", expanded=False) as stat:
+                                                try:
+                                                    st.pyplot(fig)
+                                                except Exception as e:
+                                                    st.error(f"Error plotting line chart: {e}")
+                                                    pass                    # now to deal with categorical columns
+                    categorical_columns = [col for col in filtered_columns if is_categorical_dtype(df_[col])]
+                    if categorical_columns:
+                        fig2 = plot_grouped_bar(df_, categorical_columns, date_column)
+                        with st.status(f"Date Categorical Distributions: {column}", expanded=False) as sta:
+                            try:
+                                st.pyplot(fig2)
+                            except Exception as e:
+                                st.error(f"Error plotting bar chart: {e}")
+
+        else:
+            user_text_input = right.text_input(
+                f"Substring or regex in {column}",
+            )
+            if user_text_input:
+                df_ = df_[df_[column].astype(str).str.contains(user_text_input)]
+    # write len of df after filtering with % of original
+    st.write(f"{len(df_)} rows ({len(df_) / len(df) * 100:.2f}%)")
+    return df_
+
+
+# Initialize session state
+if 'analyzers' not in st.session_state:
+    st.session_state['analyzers'] = []
+if 'col_names' not in st.session_state:
+    st.session_state['col_names'] = []
+if 'clusters' not in st.session_state:
+    st.session_state['clusters'] = {}
+if 'new_data' not in st.session_state:
+    st.session_state['new_data'] = pd.DataFrame()
+if 'dataset' not in st.session_state:
+    st.session_state['dataset'] = pd.DataFrame()
+if 'data_processed' not in st.session_state:
+    st.session_state['data_processed'] = False
+if 'stage' not in st.session_state:
+    st.session_state['stage'] = 0
+if 'filtered_data' not in st.session_state:
+    st.session_state['filtered_data'] = None
+if 'gemini_answer' not in st.session_state:
+    st.session_state['gemini_answer'] = None
+if 'parsed_responses' not in st.session_state:
+    st.session_state['parsed_responses'] = None
+if 'json_format' not in st.session_state:
+    st.session_state['json_format'] = None
+if 'api_key_valid' not in st.session_state:
+    st.session_state['api_key_valid'] = False
+if 'previous_api_key' not in st.session_state:
+    st.session_state['previous_api_key'] = None
+
+
+def load_data(file_path, key='df'):
+    return pd.read_hdf(file_path, key=key)
+
+
+datasett = st.file_uploader("Upload Raw DataFrame", type=["csv", "xlsx"])
+if datasett is not None:
+    try:
+        data = pd.read_csv(datasett) if datasett.type == "text/csv" else pd.read_excel(datasett)
+        filtered_data = filter_dataframe(data)
+        st.session_state['parsed_responses'] = filtered_data
+        st.dataframe(filtered_data)
+    except Exception as e:
+        st.error(f"An error occurred while reading the file: {e}")
+
+    col1, col2 = st.columns(2)
+    with col1:
+        columns_to_query = st.multiselect(
+        label='Select columns to analyze',
+        options=st.session_state['parsed_responses'].columns)
+    with col2:
+        COHERE_KEY = st.text_input('Cohere APIs Key', '', type='password', help="Enter your Cohere API key")
+    
+    question = st.text_input("Ask a question")
+
+    if st.session_state['parsed_responses'] is not None and question and COHERE_KEY:
+        co = cohere.Client(api_key = COHERE_KEY)
+        documents = st.session_state['parsed_responses'][columns_to_query].to_dict('records')
+        json_documents = [json.dumps(doc) for doc in documents]
+        try:
+            results = co.rerank(
+                model="rerank-english-v3.0",
+                query=question,
+                documents=json_documents,
+                top_n=5,
+                return_documents=True
+            )
+            
+            st.subheader("Reranked Results:")
+            # Create a new dataframe with reranked results
+            reranked_indices = [result.index for result in results.results]
+            reranked_scores = [result.relevance_score for result in results.results]
+            
+            reranked_df = st.session_state['parsed_responses'].iloc[reranked_indices].copy()
+            reranked_df['relevance_score'] = reranked_scores
+            reranked_df['rank'] = range(1, len(reranked_indices) + 1)
+            
+            # Set the new index to be the rank
+            reranked_df.set_index('rank', inplace=True)
+            
+            # Display the reranked dataframe
+            st.dataframe(reranked_df)
+
+            # markdown format
+            #for idx, result in enumerate(results.results, 1):
+            #    st.write(f"Result {idx}:")
+            #    st.write(f"Index: {result.index}")
+            #    st.write(f"Relevance Score: {result.relevance_score}")
+            #    st.write(f"Document: {json.loads(json_documents[result.index])}")
+            #    st.write("---")
+                
+        except Exception as e:
+            st.error(f"An error occurred during reranking: {e}")

secret_bases.csv

@@ -0,0 +1,146 @@
+,id,name,coordinates,latitude,longitude,icon
+0,10172,"633rd Air Base Wing, Joint Base LangleyEustis, VA","37.082, -76.360",37.082, -76.360,draw-shape
+1,10173,"Air Force Materiel Command, Wright Patterson AFB, Ohio","39.826, -84.048",39.826, -84.048,draw-shape
+2,10016,Air Force Office of Special Investigations,"38.871, -77.056",38.871, -77.056,draw-shape
+3,10017,Air Force Plant 42,"34.637, -118.084",34.637, -118.084,draw-shape
+4,10018,Air Force Weapons Laboratory Kirtland,"35.049, -106.609",35.049, -106.609,draw-shape
+5,10019,Area 51,"37.234, -115.806",37.234, -115.806,draw-shape
+6,10020,Area S4,"37.020, -115.787",37.020, -115.787,draw-shape
+7,10021,AT&T,"32.779, -96.808",32.779, -96.808,draw-shape
+8,10022,Avon Park Air Force Range,"27.647, -81.342",27.647, -81.342,draw-shape
+9,10023,"Azores Islands, Portugal, Lajes Field USAF","38.761, -27.091",38.761, -27.091,draw-shape
+10,10024,Barksdale AFB,"32.501, -93.663",32.501, -93.663,draw-shape
+11,10025,Battell Memorial Institute,"44.479, -73.196",44.479, -73.196,draw-shape
+12,10026,Beale Air Force Base,"39.136, -121.436",39.136, -121.436,draw-shape
+13,10027,Bechtel Corp.,"37.789, -122.396",37.789, -122.396,draw-shape
+14,10028,Bell Labs,"40.684, -74.401",40.684, -74.401,draw-shape
+15,10029,Berkeley University,"37.872, -122.259",37.872, -122.259,draw-shape
+16,10030,Blackjack Control,"36.234, -116.806",36.234, -116.806,draw-shape
+17,10031,Boeing Phantom Works,"38.676, -90.444",38.676, -90.444,draw-shape
+18,10032,"Booz-Allen and Hamilton, Inc.","38.924, -77.226",38.924, -77.226,draw-shape
+19,10033,Brooks AFB,"29.384, -98.581",29.384, -98.581,draw-shape
+20,10034,Buckley Space Force Base,"39.702, -104.751",39.702, -104.751,draw-shape
+21,10035,"C Martin Co – Gov't contractor at Dugway Proving Grounds, UT","40.197, -112.936",40.197, -112.936,draw-shape
+22,10036,Camp Peary,"37.341, -76.640",37.341, -76.640,draw-shape
+23,10037,Carswell Air Force Base,"32.769, -97.442",32.769, -97.442,draw-shape
+24,10038,China Lake Naval Air Weapons Station,"35.650, -117.669",35.650, -117.669,draw-shape
+25,10039,CIA Headquarters,"38.952, -77.146",38.952, -77.146,draw-shape
+26,10040,"CIA/160th operating under the NSC near Nashville, TN","36.167, -86.778",36.167, -86.778,draw-shape
+27,10041,Council on Foreign Relations,"40.769, -73.968",40.769, -73.968,draw-shape
+28,10042,Coyote Canyon Test Site,"35.049, -106.609",35.049, -106.609,draw-shape
+29,10043,"Crane, Indiana","38.890, -86.842",38.890, -86.842,draw-shape
+30,10044,DARPA,"38.883, -77.092",38.883, -77.092,draw-shape
+31,10045,DIA,"38.871, -77.056",38.871, -77.056,draw-shape
+32,10046,"Dugway Proving Grounds outside Provo, UT","40.197, -112.936",40.197, -112.936,draw-shape
+33,10047,Dulce NM,"36.940, -107.000",36.940, -107.000,draw-shape
+34,10048,E-Systems,"32.903, -96.461",32.903, -96.461,draw-shape
+35,10049,Eagle's Nest aka Air Force T.O.C. and The Dragon's Den,"38.871, -77.056",38.871, -77.056,draw-shape
+36,10050,Edwards North Base Complex,"34.957, -117.884",34.957, -117.884,draw-shape
+37,10051,EG&G,"42.380, -71.264",42.380, -71.264,draw-shape
+38,10052,EG&G Terminal Building,"36.080, -115.152",36.080, -115.152,draw-shape
+39,10053,Eglin Air Force Base,"30.462, -86.559",30.462, -86.559,draw-shape
+40,10054,FBI Headquarters,"38.895, -77.025",38.895, -77.025,draw-shape
+41,10055,Ford,"42.314, -83.209",42.314, -83.209,draw-shape
+42,10176,"Ford Island, Hawaii","21.364, -157.961",21.364, -157.961,draw-shape
+43,10056,Fort Benning,"32.357, -84.958",32.357, -84.958,draw-shape
+44,10057,"Fort Bragg, NC","35.141, -79.008",35.141, -79.008,draw-shape
+45,10058,Fort Chaffee Maneuver Training Center,"35.289, -94.296",35.289, -94.296,draw-shape
+46,10059,Fort Hood Army Base,"31.132, -97.781",31.132, -97.781,draw-shape
+47,10060,Fort Huachuca,"31.556, -110.346",31.556, -110.346,draw-shape
+48,10061,Fort Irwin/Raytheon ARV,"35.263, -116.687",35.263, -116.687,draw-shape
+49,10062,"Fort Jackson, South Carolina Army base","34.021, -80.897",34.021, -80.897,draw-shape
+50,10063,Fort Knox,"37.892, -85.964",37.892, -85.964,draw-shape
+51,10064,Fort Polk,"31.046, -93.208",31.046, -93.208,draw-shape
+52,10065,"Fort Sill, near Lawton, OK","34.665, -98.402",34.665, -98.402,draw-shape
+53,10165,Ft. Bliss,"31.812, -106.422",31.812, -106.422,draw-shape
+54,10066,Ft. Monmouth,"40.312, -74.045",40.312, -74.045,draw-shape
+55,10067,General Motors,"42.331, -83.046",42.331, -83.046,draw-shape
+56,10068,George AFB,"34.597, -117.384",34.597, -117.384,draw-shape
+57,10069,Goddard Spaceflight Center,"38.998, -76.852",38.998, -76.852,draw-shape
+58,10070,Grand Forks AFB,"47.961, -97.401",47.961, -97.401,draw-shape
+59,10071,Guggenheim Foundation Laboratory,"40.779, -73.960",40.779, -73.960,draw-shape
+60,10072,Hanger One Moffett Field,"37.416, -122.049",37.416, -122.049,draw-shape
+61,10073,Hanscom AFB,"42.469, -71.289",42.469, -71.289,draw-shape
+62,10074,Haystack Butte,"47.032, -111.956",47.032, -111.956,draw-shape
+63,10075,HITT Construction,"38.852, -77.322",38.852, -77.322,draw-shape
+64,10076,Holloman AFB,"32.852, -106.106",32.852, -106.106,draw-shape
+65,10077,Homestead Air Force Base,"25.488, -80.383",25.488, -80.383,draw-shape
+66,10078,Houma AFB,"29.567, -90.736",29.567, -90.736,draw-shape
+67,10079,Hughes Aircraft Company,"33.932, -118.379",33.932, -118.379,draw-shape
+68,10080,Hunter Liggett Military Reservation,"35.975, -121.229",35.975, -121.229,draw-shape
+69,10081,"Irvine, CA","33.684, -117.827",33.684, -117.827,draw-shape
+70,10082,ITT,"40.745, -73.977",40.745, -73.977,draw-shape
+71,10083,"James F Hanley Federal Bldg, Syracuse, NY","43.051, -76.150",43.051, -76.150,draw-shape
+72,10084,Jason Society,"38.895, -77.036",38.895, -77.036,draw-shape
+73,10085,John Hopkins Hospital,"39.296, -76.592",39.296, -76.592,draw-shape
+74,10086,Kelly AFB,"29.383, -98.582",29.383, -98.582,draw-shape
+75,10087,Kirtland Air Force Base,"35.049, -106.609",35.049, -106.609,draw-shape
+76,10088,Langley Air Force Base,"37.082, -76.360",37.082, -76.360,draw-shape
+77,10089,Lawrence Livermore Labs,"37.688, -121.706",37.688, -121.706,draw-shape
+78,10090,"Lewis McChord AFB, McChord, WA","47.137, -122.487",47.137, -122.487,draw-shape
+79,10091,Lockheed Martin,"39.595, -105.071",39.595, -105.071,draw-shape
+80,10093,Lockheed Martin Skunk Works,"34.637, -118.084",34.637, -118.084,draw-shape
+81,10092,Lockheed-Martin Helendale Plant,"34.742, -117.319",34.742, -117.319,draw-shape
+82,10094,Lookout Mountain Air Force Station,"34.109, -118.386",34.109, -118.386,draw-shape
+83,10095,Los Alamos National Labs,"35.838, -106.314",35.838, -106.314,draw-shape
+84,10096,"Lucerne, Switzerland secret, underground facility beneath CERN","46.234, 6.055",46.234, 6.055,draw-shape
+85,10097,Luke AFB,"33.535, -112.383",33.535, -112.383,draw-shape
+86,10098,MacDill Air Force Base,"27.849, -82.521",27.849, -82.521,draw-shape
+87,10099,Manzano Mountain Weapons Storage Facility,"34.998, -106.475",34.998, -106.475,draw-shape
+88,10100,Marshal Space Flight Center,"34.662, -86.672",34.662, -86.672,draw-shape
+89,10101,Masonic Temple,"38.895, -77.036",38.895, -77.036,draw-shape
+90,10102,Maxwell AFB,"32.380, -86.365",32.380, -86.365,draw-shape
+91,10113,Nellis AFB,"36.2, -115.0",36.2, -115.0,draw-shape
+92,10114,Nevada Test Site,"37.1, -116.1",37.1, -116.1,draw-shape
+93,10115,NORAD Cheyenne Mountain,"38.7, -104.8",38.7, -104.8,draw-shape
+94,10117,"Northrop ""Anthill""","34.8, -118.9",34.8, -118.9,draw-shape
+95,10119,Norton Air Force Base,"34.1, -117.2",34.1, -117.2,draw-shape
+96,10122,Oak Ridge National Laboratory,"35.9, -84.3",35.9, -84.3,draw-shape
+97,10123,Offutt AFB,"41.1, -95.9",41.1, -95.9,draw-shape
+98,10127,Pease Air National Guard Base,"43.1, -70.8",43.1, -70.8,draw-shape
+99,10128,Pentagon,"38.9, -77.1",38.9, -77.1,draw-shape
+100,10132,Pueblo Army Depot,"38.3, -104.3",38.3, -104.3,draw-shape
+101,10134,Red Stone Arsenal,"34.6, -86.6",34.6, -86.6,draw-shape
+102,10136,Rickenbacker Air National Guard Base,"39.8, -82.9",39.8, -82.9,draw-shape
+103,10140,Scott AFB,"38.5, -89.8",38.5, -89.8,draw-shape
+104,10144,Seymour-Johnson Air Force Base,"35.3, -77.9",35.3, -77.9,draw-shape
+105,10152,Tinker AFB,"35.4, -97.4",35.4, -97.4,draw-shape
+106,10154,Travis Air Force Base,"38.3, -121.9",38.3, -121.9,draw-shape
+107,10158,Walter Reed Hospital,"38.9, -77.0",38.9, -77.0,draw-shape
+108,10160,Wright Patterson Air Force Base,"39.8, -84.0",39.8, -84.0,draw-shape
+109,10130,Pine Gap,"-23.8, 133.7",-23.8, 133.7,draw-shape
+110,10143,"Seoul, Korea - Secret mountain facility","37.5, 127.0",37.5, 127.0,draw-shape
+111,10125,"Padang, Indonesia","-0.9, 100.4",-0.9, 100.4,draw-shape
+112,10171,Peasemore,"51.5, -1.3",51.5, -1.3,draw-shape
+113,10102,Maxwell AFB,"32.4, -86.4",32.4, -86.4,draw-shape
+114,10103,McClellan Air Force Base,"38.7, -121.4",38.7, -121.4,draw-shape
+115,10104,McDonald Douglas Llano Plant,"34.5, -117.8",34.5, -117.8,draw-shape
+116,10105,Miramar Naval Base,"32.9, -117.1",32.9, -117.1,draw-shape
+117,10106,MIT,"42.4, -71.1",42.4, -71.1,draw-shape
+118,10169,Mount Hough,"39.9, -120.9",39.9, -120.9,draw-shape
+119,10109,NASA Ames Research Center,"37.4, -122.1",37.4, -122.1,draw-shape
+120,10110,NASA Johnson Space Center,"29.6, -95.1",29.6, -95.1,draw-shape
+121,10111,Naval Air Station,"44.9, -66.9",44.9, -66.9,draw-shape
+122,10112,Naval Station Great Lakes,"42.3, -87.8",42.3, -87.8,draw-shape
+123,10113,Nellis AFB,"36.2, -115.0",36.2, -115.0,draw-shape
+124,10114,Nevada Test Site,"37.1, -116.1",37.1, -116.1,draw-shape
+125,10115,NORAD Cheyenne Mountain,"38.7, -104.8",38.7, -104.8,draw-shape
+126,10117,"Northrop ""Anthill""","34.8, -118.9",34.8, -118.9,draw-shape
+127,10119,Norton Air Force Base,"34.1, -117.2",34.1, -117.2,draw-shape
+128,10122,Oak Ridge National Laboratory,"35.9, -84.3",35.9, -84.3,draw-shape
+129,10123,Offutt AFB,"41.1, -95.9",41.1, -95.9,draw-shape
+130,10127,Pease Air National Guard Base,"43.1, -70.8",43.1, -70.8,draw-shape
+131,10128,Pentagon,"38.9, -77.1",38.9, -77.1,draw-shape
+132,10132,Pueblo Army Depot,"38.3, -104.3",38.3, -104.3,draw-shape
+133,10134,Red Stone Arsenal,"34.6, -86.6",34.6, -86.6,draw-shape
+134,10136,Rickenbacker Air National Guard Base,"39.8, -82.9",39.8, -82.9,draw-shape
+135,10140,Scott AFB,"38.5, -89.8",38.5, -89.8,draw-shape
+136,10144,Seymour-Johnson Air Force Base,"35.3, -77.9",35.3, -77.9,draw-shape
+137,10152,Tinker AFB,"35.4, -97.4",35.4, -97.4,draw-shape
+138,10154,Travis Air Force Base,"38.3, -121.9",38.3, -121.9,draw-shape
+139,10158,Walter Reed Hospital,"38.9, -77.0",38.9, -77.0,draw-shape
+140,10160,Wright Patterson Air Force Base,"39.8, -84.0",39.8, -84.0,draw-shape
+141,10130,Pine Gap,"-23.8, 133.7",-23.8, 133.7,draw-shape
+142,10143,"Seoul, Korea - Secret mountain facility","37.5, 127.0",37.5, 127.0,draw-shape
+143,10125,"Padang, Indonesia","-0.9, 100.4",-0.9, 100.4,draw-shape
+144,10171,Peasemore,"51.5, -1.3",51.5, -1.3,draw-shape

uap_analyzer.py

@@ -0,0 +1,1010 @@
+import pandas as pd
+import numpy as np
+from sklearn.decomposition import PCA
+from sklearn.cluster import KMeans
+from cuml.manifold import umap
+from cuml.cluster import hdbscan
+import plotly.graph_objects as go
+from sentence_transformers import SentenceTransformer
+import torch
+with torch.no_grad():
+    embed_model = SentenceTransformer('embaas/sentence-transformers-e5-large-v2')
+    embed_model.to('cuda')
+from sentence_transformers.util import pytorch_cos_sim, pairwise_cos_sim
+#from stqdm.notebook import stqdm
+#stqdm.pandas()
+import logging
+import pandas as pd
+import numpy as np
+from sklearn.decomposition import PCA
+from sklearn.cluster import KMeans
+import plotly.graph_objects as go
+import plotly.express as px
+from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
+import numpy as np
+from Levenshtein import distance
+import logging
+from sklearn.metrics import confusion_matrix
+import seaborn as sns
+import matplotlib.pyplot as plt
+import xgboost as xgb
+from xgboost import plot_importance
+import matplotlib.pyplot as plt
+from sklearn.metrics import accuracy_score, confusion_matrix
+from scipy.stats import chi2_contingency
+import matplotlib.pyplot as plt
+import seaborn as sns
+from statsmodels.graphics.mosaicplot import mosaic
+import pickle
+import pandas as pd
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import confusion_matrix
+import seaborn as sns
+import matplotlib.pyplot as plt
+import xgboost as xgb
+from xgboost import plot_importance
+import matplotlib.pyplot as plt
+from sklearn.metrics import accuracy_score, confusion_matrix
+from scipy.stats import chi2_contingency
+import matplotlib.pyplot as plt
+import seaborn as sns
+from statsmodels.graphics.mosaicplot import mosaic
+from statsmodels.api import stats
+import os
+import time
+import concurrent.futures
+from requests.exceptions import HTTPError
+from stqdm import stqdm
+stqdm.pandas()
+import json
+import pandas as pd
+from openai import OpenAI
+import numpy as np
+import matplotlib.pyplot as plt
+import squarify
+import matplotlib.colors as mcolors
+import textwrap
+import pandas as pd
+import streamlit as st
+st.set_option('deprecation.showPyplotGlobalUse', False)
+
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+
+class UAPAnalyzer:
+    """
+    A class for analyzing and clustering textual data within a pandas DataFrame using 
+    Natural Language Processing (NLP) techniques and machine learning models.
+    
+    Attributes:
+        data (pd.DataFrame): The dataset containing textual data for analysis.
+        column (str): The name of the column in the DataFrame to be analyzed.
+        embeddings (np.ndarray): The vector representations of textual data.
+        reduced_embeddings (np.ndarray): The dimensionality-reduced embeddings.
+        cluster_labels (np.ndarray): The labels assigned to each data point after clustering.
+        cluster_terms (list): The list of terms associated with each cluster.
+        tfidf_matrix (sparse matrix): The Term Frequency-Inverse Document Frequency (TF-IDF) matrix.
+        models (dict): A dictionary to store trained machine learning models.
+        evaluations (dict): A dictionary to store evaluation results of models.
+        data_nums (pd.DataFrame): The DataFrame with numerical encoding of categorical data.
+    """
+
+    def __init__(self, data, column, has_embeddings=False):
+        """
+        Initializes the UAPAnalyzer with a dataset and a specified column for analysis.
+        
+        Args:
+            data (pd.DataFrame): The dataset for analysis.
+            column (str): The column within the dataset to analyze.
+        """
+        assert isinstance(data, pd.DataFrame), "Data must be a pandas DataFrame"
+        assert column in data.columns, f"Column '{column}' not found in DataFrame"
+        self.has_embeddings = has_embeddings        
+        self.data = data
+        self.column = column
+        self.embeddings = None
+        self.reduced_embeddings = None
+        self.cluster_labels = None
+        self.cluster_names = None
+        self.cluster_terms = None 
+        self.cluster_terms_embeddings = None
+        self.tfidf_matrix = None
+        self.models = {}  # To store trained models
+        self.evaluations = {}  # To store evaluation results
+        self.data_nums = None  # Encoded numerical data
+        self.x_train = None
+        self.y_train = None
+        self.x_test = None
+        self.y_test = None
+        self.preds = None
+        self.new_dataset = None
+        self.model = embed_model
+        self.model = self.model.to('cuda')
+        #self.cluster_names_ = pd.DataFrame()
+
+        logging.info("UAPAnalyzer initialized")
+
+    def preprocess_data(self, trim=False, has_embeddings=False, top_n=32,):
+        """
+        Preprocesses the data by optionally trimming the dataset to include only the top N labels and extracting embeddings.
+        
+        Args:
+            trim (bool): Whether to trim the dataset to include only the top N labels.
+            top_n (int): The number of top labels to retain if trimming is enabled.
+        """
+        logging.info("Preprocessing data")
+
+        # if trim is True
+        if trim:
+            # Identify the top labels based on value counts
+            top_labels = self.data[self.column].value_counts().nlargest(top_n).index.tolist()
+            # Revise the column data, setting values to 'Other' if they are not in the top labels
+            self.data[f'{self.column}_revised'] = np.where(self.data[self.column].isin(top_labels), self.data[self.column], 'Other')
+        # Convert the column data to string type before passing to _extract_embeddings
+        # This is useful especially if the data type of the column is not originally string
+        string_data = self.data[f'{self.column}'].astype(str)
+        # Extract embeddings from the revised and string-converted column data
+        if has_embeddings:
+            self.embeddings = self.data['embeddings'].to_list()
+        else:
+            self.embeddings = self._extract_embeddings(string_data)
+        logging.info("Data preprocessing complete")
+
+
+    def _extract_embeddings(self, data_column):
+        """
+        Extracts embeddings from the given data column.
+        
+        Args:
+            data_column (pd.Series): The column from which to extract embeddings.
+        
+        Returns:
+            np.ndarray: The extracted embeddings.
+        """
+        logging.info("Extracting embeddings")
+        # convert to str
+        return embed_model.encode(data_column.tolist(), show_progress_bar=True)
+
+    def reduce_dimensionality(self, method='UMAP', n_components=2, **kwargs):
+        """
+        Reduces the dimensionality of embeddings using specified method.
+        
+        Args:
+            method (str): The dimensionality reduction method to use ('UMAP' or 'PCA').
+            n_components (int): The number of dimensions to reduce to.
+            **kwargs: Additional keyword arguments for the dimensionality reduction method.
+        """
+        logging.info(f"Reducing dimensionality using {method}")
+        if method == 'UMAP':
+            reducer = umap.UMAP(n_components=n_components, **kwargs)
+        elif method == 'PCA':
+            reducer = PCA(n_components=n_components)
+        else:
+            raise ValueError("Unsupported dimensionality reduction method")
+        
+        self.reduced_embeddings = reducer.fit_transform(self.embeddings)
+        logging.info(f"Dimensionality reduced using {method}")
+
+    def cluster_data(self, method='HDBSCAN', **kwargs):
+        """
+        Clusters the reduced dimensionality data using the specified clustering method.
+        
+        Args:
+            method (str): The clustering method to use ('HDBSCAN' or 'KMeans').
+            **kwargs: Additional keyword arguments for the clustering method.
+        """
+        logging.info(f"Clustering data using {method}")
+        if method == 'HDBSCAN':
+            clusterer = hdbscan.HDBSCAN(**kwargs)
+        elif method == 'KMeans':
+            clusterer = KMeans(**kwargs)
+        else:
+            raise ValueError("Unsupported clustering method")
+        
+        clusterer.fit(self.reduced_embeddings)
+        self.cluster_labels = clusterer.labels_
+        logging.info(f"Data clustering complete using {method}")
+
+        
+    def get_tf_idf_clusters(self, top_n=2):
+        """
+        Names clusters using the most frequent terms based on TF-IDF analysis.
+
+        Args:
+            top_n (int): The number of top terms to consider for naming each cluster.
+        """
+        logging.info("Naming clusters based on top TF-IDF terms.")
+
+        # Ensure data has been clustered
+        assert self.cluster_labels is not None, "Data has not been clustered yet."
+        vectorizer = TfidfVectorizer(max_features=1000, stop_words='english')
+
+        # Fit the vectorizer to the text data and transform it into a TF-IDF matrix
+        tfidf_matrix = vectorizer.fit_transform(self.data[f'{self.column}'].astype(str))
+
+        # Initialize an empty list to store the cluster terms
+        self.cluster_terms = []
+
+        for cluster_id in np.unique(self.cluster_labels):
+            # Skip noise if present (-1 in HDBSCAN)
+            if cluster_id == -1:
+                continue
+
+            # Find indices of documents in the current cluster
+            indices = np.where(self.cluster_labels == cluster_id)[0]
+
+            # Compute the mean TF-IDF score for each term in the cluster
+            cluster_tfidf_mean = np.mean(tfidf_matrix[indices], axis=0)
+
+            # Use the matrix directly for indexing if it does not support .toarray()
+            # Ensure it's in a format that supports indexing, convert if necessary
+            if hasattr(cluster_tfidf_mean, "toarray"):
+                dense_mean = cluster_tfidf_mean.toarray().flatten()
+            else:
+                dense_mean = np.asarray(cluster_tfidf_mean).flatten()
+
+            # Get the indices of the top_n terms
+            top_n_indices = np.argsort(dense_mean)[-top_n:]
+
+            # Get the corresponding terms for these top indices
+            terms = vectorizer.get_feature_names_out()
+            top_terms = [terms[i] for i in top_n_indices]
+
+            # Join the top_n terms with a hyphen
+            cluster_name = '-'.join(top_terms)
+
+            # Append the cluster name to the list
+            self.cluster_terms.append(cluster_name)
+
+        # Convert the list of cluster terms to a categorical data type
+        self.cluster_terms = pd.Categorical(self.cluster_terms)
+        logging.info("Cluster naming completed.")
+
+    def merge_similar_clusters(self, distance='cosine', char_diff_threshold = 3, similarity_threshold = 0.92, embeddings = 'SBERT'):
+        """
+        Merges similar clusters based on cosine similarity of their associated terms.
+        
+        Args:
+            similarity_threshold (float): The similarity threshold above which clusters are considered similar enough to merge.
+        """
+        from collections import defaultdict
+        logging.info("Merging similar clusters")
+
+        # A mapping from cluster names to a set of cluster names to be merged
+        merge_mapping = defaultdict(set)
+        merge_labels = defaultdict(set)
+
+        if distance == 'levenshtein':
+            distances = {}
+            for i, name1 in enumerate(self.cluster_terms):
+                for j, name2 in enumerate(self.cluster_terms[i + 1:], start=i + 1):
+                    dist = distance(name1, name2)
+                    if dist <= char_diff_threshold:
+                        logging.info(f"Merging '{name2}' into '{name1}'")
+                        merge_mapping[name1].add(name2)
+
+        elif distance == 'cosine':
+            self.cluster_terms_embeddings = embed_model.encode(self.cluster_terms)
+            cos_sim_matrix = pytorch_cos_sim(self.cluster_terms_embeddings, self.cluster_terms_embeddings)
+            for i, name1 in enumerate(self.cluster_terms):
+                for j, name2 in enumerate(self.cluster_terms[i + 1:], start=i + 1):
+                    if cos_sim_matrix[i][j] > similarity_threshold:
+                        #st.write(f"Merging cluster '{name2}' into cluster '{name1}' based on cosine similarity")
+                        logging.info(f"Merging cluster '{name2}' into cluster '{name1}' based on cosine similarity")
+                        merge_mapping[name1].add(name2)
+
+
+        # Flatten the merge mapping to a simple name change mapping
+        name_change_mapping = {}
+        for cluster_name, merges in merge_mapping.items():
+            for merge_name in merges:
+                name_change_mapping[merge_name] = cluster_name
+
+        # Update cluster labels based on name changes
+        updated_cluster_terms = []
+        original_to_updated_index = {}
+        for i, name in enumerate(self.cluster_terms):
+            updated_name = name_change_mapping.get(name, name)
+            if updated_name not in updated_cluster_terms:
+                updated_cluster_terms.append(updated_name)
+                original_to_updated_index[i] = len(updated_cluster_terms) - 1
+            else:
+                updated_index = updated_cluster_terms.index(updated_name)
+                original_to_updated_index[i] = updated_index
+
+        self.cluster_terms = updated_cluster_terms  # Update cluster terms with merged names
+        self.clusters_labels = np.array([original_to_updated_index[label] for label in self.cluster_labels])
+
+
+        # Update cluster labels according to the new index mapping
+        # self.cluster_labels = np.array([original_to_updated_index[label] if label in original_to_updated_index else -1 for label in self.cluster_labels])
+        # self.cluster_terms = [self.cluster_terms[original_to_updated_index[label]] if label != -1 else 'Noise' for label in self.cluster_labels]
+
+        # Log the total number of merges
+        total_merges = sum(len(merges) for merges in merge_mapping.values())
+        logging.info(f"Total clusters merged: {total_merges}")
+
+        unique_labels = np.unique(self.cluster_labels)
+        label_to_index = {label: index for index, label in enumerate(unique_labels)}
+        self.cluster_labels = np.array([label_to_index[label] for label in self.cluster_labels])
+        self.cluster_terms = [self.cluster_terms[label_to_index[label]] for label in self.cluster_labels]
+
+    def merge_similar_clusters2(self, distance='cosine', char_diff_threshold=3, similarity_threshold=0.92):
+        logging.info("Merging similar clusters based on distance: {}".format(distance))
+        from collections import defaultdict
+        merge_mapping = defaultdict(set)
+
+        if distance == 'levenshtein':
+            for i, name1 in enumerate(self.cluster_terms):
+                for j, name2 in enumerate(self.cluster_terms[i + 1:], start=i + 1):
+                    dist = distance(name1, name2)
+                    if dist <= char_diff_threshold:
+                        merge_mapping[name1].add(name2)
+                        logging.info(f"Merging '{name2}' into '{name1}' based on Levenshtein distance")
+
+        elif distance == 'cosine':
+            if self.cluster_terms_embeddings is None:
+                self.cluster_terms_embeddings = embed_model.encode(self.cluster_terms)
+            cos_sim_matrix = pytorch_cos_sim(self.cluster_terms_embeddings, self.cluster_terms_embeddings)
+            for i in range(len(self.cluster_terms)):
+                for j in range(i + 1, len(self.cluster_terms)):
+                    if cos_sim_matrix[i][j] > similarity_threshold:
+                        merge_mapping[self.cluster_terms[i]].add(self.cluster_terms[j])
+                        #st.write(f"Merging cluster '{self.cluster_terms[j]}' into cluster '{self.cluster_terms[i]}'")
+                        logging.info(f"Merging cluster '{self.cluster_terms[j]}' into cluster '{self.cluster_terms[i]}'")
+
+        self._update_cluster_terms_and_labels(merge_mapping)
+
+    def _update_cluster_terms_and_labels(self, merge_mapping):
+        # Flatten the merge mapping to a simple name change mapping
+        name_change_mapping = {old: new for new, olds in merge_mapping.items() for old in olds}
+        # Update cluster terms and labels
+        unique_new_terms = list(set(name_change_mapping.values()))
+        # replace the old terms with the new terms (name2) otherwise, keep the old terms (name1)
+        # self.cluster_terms = [name_change_mapping.get(term, term) for term in self.cluster_terms]
+        # self.cluster_labels = np.array([unique_new_terms.index(term) if term in unique_new_terms else term for term in self.cluster_terms])
+        self.cluster_terms = [name_change_mapping.get(term, term) for term in self.cluster_terms]
+        self.cluster_labels = [unique_new_terms.index(term) if term in unique_new_terms else -1 for term in self.cluster_terms]
+
+        logging.info(f"Total clusters merged: {len(merge_mapping)}")
+
+
+    def cluster_levenshtein(self, cluster_terms, cluster_labels, char_diff_threshold=3):
+        from Levenshtein import distance  # Make sure to import the correct distance function
+
+        merge_map = {}
+        # Iterate over term pairs and decide on merging based on the distance
+        for idx, term1 in enumerate(cluster_terms):
+            for jdx, term2 in enumerate(cluster_terms):
+                if idx < jdx and distance(term1, term2) <= char_diff_threshold:  
+                    labels_to_merge = [label for label, term_index in enumerate(cluster_labels) if term_index == jdx]
+                    for label in labels_to_merge:
+                        merge_map[label] = idx  # Map the label to use the term index of term1
+                    logging.info(f"Merging '{term2}' into '{term1}'")
+                    st.write(f"Merging '{term2}' into '{term1}'")
+        # Update the cluster labels
+        updated_cluster_labels = [merge_map.get(label, label) for label in cluster_labels]
+        # Update string labels to reflect merged labels
+        updated_string_labels = [cluster_terms[label] for label in updated_cluster_labels]
+        return updated_string_labels
+
+    def cluster_cosine(self, cluster_terms, cluster_labels, similarity_threshold):
+        from sklearn.metrics.pairwise import cosine_similarity
+        cluster_terms_embeddings = embed_model.encode(cluster_terms)
+        # Compute cosine similarity matrix in a vectorized form
+        cos_sim_matrix = cosine_similarity(cluster_terms_embeddings, cluster_terms_embeddings)
+
+        merge_map = {}
+        n_terms = len(cluster_terms)
+        # Iterate only over upper triangular matrix excluding diagonal to avoid redundant computations and self-comparison
+        for idx in range(n_terms):
+            for jdx in range(idx + 1, n_terms):
+                if cos_sim_matrix[idx, jdx] >= similarity_threshold:
+                    labels_to_merge = [label for label, term_index in enumerate(cluster_labels) if term_index == jdx]
+                    for label in labels_to_merge:
+                        merge_map[label] = idx
+                    st.write(f"Merging '{cluster_terms[jdx]}' into '{cluster_terms[idx]}'")
+                    logging.info(f"Merging '{cluster_terms[jdx]}' into '{cluster_terms[idx]}'")
+        # Update the cluster labels
+        updated_cluster_labels = [merge_map.get(label, label) for label in cluster_labels]
+        # Update string labels to reflect merged labels
+        updated_string_labels = [cluster_terms[label] for label in updated_cluster_labels]
+        # make a dataframe with index, cluster label and cluster term
+        return updated_string_labels
+
+    def merge_similar_clusters(self, cluster_terms, cluster_labels, distance_type='cosine', char_diff_threshold=3, similarity_threshold=0.92):
+        if distance_type == 'levenshtein':
+            return self.cluster_levenshtein(cluster_terms, cluster_labels, char_diff_threshold)
+        elif distance_type == 'cosine':
+            return self.cluster_cosine(cluster_terms, cluster_labels, similarity_threshold)
+
+    def plot_embeddings2(self, title=None):
+        assert self.reduced_embeddings is not None, "Dimensionality reduction has not been performed yet."
+        assert self.cluster_terms is not None, "Cluster TF-IDF analysis has not been performed yet."
+
+        logging.info("Plotting embeddings with TF-IDF colors")
+
+        fig = go.Figure()
+
+        unique_cluster_terms = np.unique(self.cluster_terms)
+
+        for cluster_term in unique_cluster_terms:
+            if cluster_term != 'Noise':
+                indices = np.where(np.array(self.cluster_terms) == cluster_term)[0]
+
+                # Plot points in the current cluster
+                fig.add_trace(
+                    go.Scatter(
+                        x=self.reduced_embeddings[indices, 0],
+                        y=self.reduced_embeddings[indices, 1],
+                        mode='markers',
+                        marker=dict(
+                            size=5,
+                            opacity=0.8,
+                        ),
+                        name=cluster_term,
+                        text=self.data[f'{self.column}'].iloc[indices], 
+                        hoverinfo='text',
+                    )
+                )
+            else:
+                # Plot noise points differently if needed
+                fig.add_trace(
+                    go.Scatter(
+                        x=self.reduced_embeddings[indices, 0],
+                        y=self.reduced_embeddings[indices, 1],
+                        mode='markers',
+                        marker=dict(
+                            size=5,
+                            opacity=0.5,
+                            color='grey'
+                        ),
+                        name='Noise',
+                        text=[self.data[f'{self.column}'][i] for i in indices],  # Adjusted for potential pandas use
+                        hoverinfo='text',
+                    )
+                )
+            # else:
+            #     indices = np.where(np.array(self.cluster_terms) == 'Noise')[0]
+
+            #     # Plot noise points
+            #     fig.add_trace(
+            #         go.Scatter(
+            #             x=self.reduced_embeddings[indices, 0],
+            #             y=self.reduced_embeddings[indices, 1],
+            #             mode='markers',
+            #             marker=dict(
+            #                 size=5,
+            #                 opacity=0.8,
+            #             ),
+            #             name='Noise',
+            #             text=self.data[f'{self.column}'].iloc[indices],
+            #             hoverinfo='text',
+            #         )
+            #     )
+
+        fig.update_layout(title=title, showlegend=True, legend_title_text='Top TF-IDF Terms')
+        #return fig
+        st.plotly_chart(fig, use_container_width=True)  
+       #fig.show()
+        #logging.info("Embeddings plotted with TF-IDF colors")
+
+    def plot_embeddings3(self, title=None):
+        assert self.reduced_embeddings is not None, "Dimensionality reduction has not been performed yet."
+        assert self.cluster_terms is not None, "Cluster TF-IDF analysis has not been performed yet."
+
+        logging.info("Plotting embeddings with TF-IDF colors")
+
+        fig = go.Figure()
+
+        unique_cluster_terms = np.unique(self.cluster_terms)
+
+        terms_order = {term: i for i, term in enumerate(np.unique(self.cluster_terms, return_index=True)[0])}
+        #indices = np.argsort([terms_order[term] for term in self.cluster_terms])
+
+        # Handling color assignment, especially for noise
+        colors = {term: ('grey' if term == 'Noise' else None) for term in unique_cluster_terms}
+        color_map = px.colors.qualitative.Plotly  # Default color map from Plotly Express for consistency
+
+        # Apply a custom color map, handling 'Noise' specifically
+        color_idx = 0
+        for cluster_term in unique_cluster_terms:
+            indices = np.where(np.array(self.cluster_terms) == cluster_term)[0]
+            if cluster_term != 'Noise':
+                marker_color = color_map[color_idx % len(color_map)]
+                color_idx += 1
+            else:
+                marker_color = 'grey'
+
+            fig.add_trace(
+                go.Scatter(
+                    x=self.reduced_embeddings[indices, 0],
+                    y=self.reduced_embeddings[indices, 1],
+                    mode='markers',
+                    marker=dict(
+                        size=5,
+                        opacity=(0.5 if cluster_term == 'Noise' else 0.8),
+                        color=marker_color
+                    ),
+                    name=cluster_term,
+                    text=self.data[f'{self.column}'].iloc[indices],
+                    hoverinfo='text'
+                )
+            )
+        fig.data = sorted(fig.data, key=lambda trace: terms_order[trace.name])
+        fig.update_layout(title=title if title else "Embeddings Visualized", showlegend=True, legend_title_text='Top TF-IDF Terms')
+        st.plotly_chart(fig, use_container_width=True)
+
+
+    def plot_embeddings(self, title=None):
+        """
+        Plots the reduced dimensionality embeddings with clusters indicated.
+        
+        Args:
+            title (str): The title of the plot.
+        """
+        # Ensure dimensionality reduction and TF-IDF based cluster naming have been performed
+        assert self.reduced_embeddings is not None, "Dimensionality reduction has not been performed yet."
+        assert self.cluster_terms is not None, "Cluster TF-IDF analysis has not been performed yet."
+
+        logging.info("Plotting embeddings with TF-IDF colors")
+        
+        fig = go.Figure()
+        
+        #for i, term in enumerate(self.cluster_terms):
+            # Indices of points in the current cluster
+        #unique_cluster_ids = np.unique(self.cluster_labels[self.cluster_labels != -1])  # Exclude noise
+        unique_cluster_terms = np.unique(self.cluster_terms)
+        unique_cluster_labels = np.unique(self.cluster_labels)
+            
+        for i, (cluster_id, cluster_terms) in enumerate(zip(unique_cluster_labels, unique_cluster_terms)):
+            indices = np.where(self.cluster_labels == cluster_id)[0]
+            #indices = np.where(self.cluster_labels == i)[0]
+            
+            # Plot points in the current cluster
+            fig.add_trace(
+                go.Scatter(
+                    x=self.reduced_embeddings[indices, 0],
+                    y=self.reduced_embeddings[indices, 1],
+                    mode='markers',
+                    marker=dict(
+                        #color=i,
+                        #colorscale='rainbow',
+                        size=5,
+                        opacity=0.8,
+                    ),
+                    name=cluster_terms,
+                    text=self.data[f'{self.column}'].iloc[indices],
+                    hoverinfo='text',
+                )
+            )
+            
+        
+        fig.update_layout(title=title, showlegend=True, legend_title_text='Top TF-IDF Terms')
+        st.plotly_chart(fig, use_container_width=True)
+        logging.info("Embeddings plotted with TF-IDF colors")
+
+    def plot_embeddings4(self, title=None, cluster_terms=None, cluster_labels=None, reduced_embeddings=None, column=None, data=None):
+        """
+        Plots the reduced dimensionality embeddings with clusters indicated.
+        
+        Args:
+            title (str): The title of the plot.
+        """
+        # Ensure dimensionality reduction and TF-IDF based cluster naming have been performed
+        assert reduced_embeddings is not None, "Dimensionality reduction has not been performed yet."
+        assert cluster_terms is not None, "Cluster TF-IDF analysis has not been performed yet."
+
+        logging.info("Plotting embeddings with TF-IDF colors")
+        
+        fig = go.Figure()
+        
+        # Determine unique cluster IDs and terms, and ensure consistent color mapping
+        unique_cluster_ids = np.unique(cluster_labels)
+        unique_cluster_terms = [cluster_terms[i] for i in unique_cluster_ids]#if i != -1]  # Exclude noise by ID
+
+        color_map = px.colors.qualitative.Plotly  # Using Plotly Express's qualitative colors for consistency
+        color_idx = 0
+        
+        # Map each cluster ID to a color
+        cluster_colors = {}
+        for cid in unique_cluster_ids:
+            #if cid != -1:  # Exclude noise
+                cluster_colors[cid] = color_map[color_idx % len(color_map)]
+                color_idx += 1
+            #else:
+            #    cluster_colors[cid] = 'grey'  # Noise or outliers in grey
+
+        for cluster_id, cluster_term in zip(unique_cluster_ids, unique_cluster_terms):
+            indices = np.where(cluster_labels == cluster_id)[0]
+            fig.add_trace(
+                go.Scatter(
+                    x=reduced_embeddings[indices, 0],
+                    y=reduced_embeddings[indices, 1],
+                    mode='markers',
+                    marker=dict(
+                        color=cluster_colors[cluster_id],
+                        size=5,
+                        opacity=0.8#if cluster_id != -1 else 0.5,
+                    ),
+                    name=cluster_term,
+                    text=data[column].iloc[indices],  # Use the original column for hover text
+                    hoverinfo='text',
+                )
+            )
+            
+        fig.update_layout(
+            title=title if title else "Embeddings Visualized",
+            showlegend=True,
+            legend_title_text='Top TF-IDF Terms',
+            legend=dict(
+                traceorder='normal',  # 'normal' or 'reversed'; ensures that traces appear in the order they are added
+                itemsizing='constant'
+            )
+        )
+        st.plotly_chart(fig, use_container_width=True)
+        logging.info("Embeddings plotted with TF-IDF colors")
+
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+
+def analyze_and_predict(data, analyzers, col_names):
+    """
+    Performs analysis on the data using provided analyzers and makes predictions on specified columns.
+
+    Args:
+        data (pd.DataFrame): The dataset for analysis.
+        analyzers (list): A list of UAPAnalyzer instances.
+        col_names (list): Column names to be analyzed and predicted.
+    """
+    new_data = pd.DataFrame()
+    for i, (column, analyzer) in enumerate(zip(col_names, analyzers)):
+        new_data[f'Analyzer_{column}'] = analyzer.__dict__['cluster_terms']
+        logging.info(f"Cluster terms extracted for {column}")
+
+    new_data = new_data.fillna('null').astype('category')
+    data_nums = new_data.apply(lambda x: x.cat.codes)
+
+    for col in data_nums.columns:
+        try:
+            categories = new_data[col].cat.categories
+            x_train, x_test, y_train, y_test = train_test_split(data_nums.drop(columns=[col]), data_nums[col], test_size=0.2, random_state=42)
+            bst, accuracy, preds = train_xgboost(x_train, y_train, x_test, y_test, len(categories))
+            plot_results(new_data, bst, x_test, y_test, preds, categories, accuracy, col)
+        except Exception as e:
+            logging.error(f"Error processing {col}: {e}")
+    return new_data
+
+def train_xgboost(x_train, y_train, x_test, y_test, num_classes):
+    """
+    Trains an XGBoost model and evaluates its performance.
+
+    Args:
+        x_train (pd.DataFrame): Training features.
+        y_train (pd.Series): Training labels.
+        x_test (pd.DataFrame): Test features.
+        y_test (pd.Series): Test labels.
+        num_classes (int): The number of unique classes in the target variable.
+
+    Returns:
+        bst (Booster): The trained XGBoost model.
+        accuracy (float): The accuracy of the model on the test set.
+    """
+    dtrain = xgb.DMatrix(x_train, label=y_train, enable_categorical=True)
+    dtest = xgb.DMatrix(x_test, label=y_test)
+
+    params = {'device':'cuda', 'objective': 'multi:softmax', 'num_class': num_classes, 'max_depth': 6, 'eta': 0.3}
+    num_round = 100
+    bst = xgb.train(dtrain=dtrain, params=params, num_boost_round=num_round)
+    preds = bst.predict(dtest)
+    accuracy = accuracy_score(y_test, preds)
+
+    logging.info(f"XGBoost trained with accuracy: {accuracy:.2f}")
+    return bst, accuracy, preds
+
+def plot_results(new_data, bst, x_test, y_test, preds, categories, accuracy, col):
+    """
+    Plots the feature importance, confusion matrix, and contingency table.
+
+    Args:
+        bst (Booster): The trained XGBoost model.
+        x_test (pd.DataFrame): Test features.
+        y_test (pd.Series): Test labels.
+        preds (np.array): Predictions made by the model.
+        categories (Index): Category names for the target variable.
+        accuracy (float): The accuracy of the model on the test set.
+        col (str): The target column name being analyzed and predicted.
+    """
+    fig, axs = plt.subplots(1, 3, figsize=(25, 5), dpi=300)
+    fig.suptitle(f'{col.split(sep=".")[-1]} prediction', fontsize=35)
+
+    plot_importance(bst, ax=axs[0], importance_type='gain', show_values=False)
+    conf_matrix = confusion_matrix(y_test, preds)
+    sns.heatmap(conf_matrix, annot=True, fmt='g', cmap='Blues', xticklabels=categories, yticklabels=categories, ax=axs[1])
+    axs[1].set_title(f'Confusion Matrix\nAccuracy: {accuracy * 100:.2f}%')
+    # make axes rotated
+    axs[1].set_yticklabels(axs[1].get_yticklabels(), rotation=30, ha='right')
+    sorted_features = sorted(bst.get_score(importance_type="gain").items(), key=lambda x: x[1], reverse=True)
+    # The most important feature is the first element in the sorted list
+    most_important_feature = sorted_features[0][0]
+    # Create a contingency table
+    contingency_table = pd.crosstab(new_data[col], new_data[most_important_feature])
+
+    # resid pearson is used to calculate the residuals, which 
+    table = stats.Table(contingency_table).resid_pearson
+    #print(table)
+    # Perform the chi-squared test
+    chi2, p, dof, expected = chi2_contingency(contingency_table)
+    # Print the results
+    print(f"Chi-squared test for {col} and {most_important_feature}: p-value = {p}")
+    
+    sns.heatmap(table, annot=True, cmap='Greens', ax=axs[2])
+    # make axis rotated
+    axs[2].set_yticklabels(axs[2].get_yticklabels(), rotation=30, ha='right')
+    axs[2].set_title(f'Contingency Table between {col.split(sep=".")[-1]} and {most_important_feature.split(sep=".")[-1]}\np-value = {p}')    
+
+    plt.tight_layout()
+    #plt.savefig(f"{col}_{accuracy:.2f}_prediction_XGB.jpeg", dpi=300)
+    return plt
+
+def cramers_v(confusion_matrix):
+    """Calculate Cramer's V statistic for categorical-categorical association."""
+    chi2 = chi2_contingency(confusion_matrix)[0]
+    n = confusion_matrix.sum().sum()
+    phi2 = chi2 / n
+    r, k = confusion_matrix.shape
+    phi2corr = max(0, phi2 - ((k-1)*(r-1))/(n-1))
+    r_corr = r - ((r-1)**2)/(n-1)
+    k_corr = k - ((k-1)**2)/(n-1)
+    return np.sqrt(phi2corr / min((k_corr-1), (r_corr-1)))
+
+def plot_cramers_v_heatmap(data, significance_level=0.05):
+    """Plot heatmap of Cramer's V statistic for each pair of categorical variables in a DataFrame."""
+    # Initialize a DataFrame to store Cramer's V values
+    cramers_v_df = pd.DataFrame(index=data.columns, columns=data.columns, data=np.nan)
+    
+    # Compute Cramer's V for each pair of columns
+    for col1 in data.columns:
+        for col2 in data.columns:
+            if col1 != col2:  # Avoid self-comparison
+                confusion_matrix = pd.crosstab(data[col1], data[col2])
+                chi2, p, dof, expected = chi2_contingency(confusion_matrix)
+                # Check if the p-value is less than the significance level
+                #if p < significance_level:
+                #    cramers_v_df.at[col1, col2] = cramers_v(confusion_matrix)
+                # alternatively, you can use the following line to include all pairs
+                cramers_v_df.at[col1, col2] = cramers_v(confusion_matrix)
+    
+    # Plot the heatmap
+    plt.figure(figsize=(12, 10), dpi=200)
+    mask = np.triu(np.ones_like(cramers_v_df, dtype=bool))  # Mask for the upper triangle
+    # make a max and min of the cmap
+    sns.heatmap(cramers_v_df, annot=True, fmt=".2f", cmap='coolwarm', cbar=True, mask=mask, square=True)
+    plt.title(f"Heatmap of Cramér's V (p < {significance_level})")
+    return plt
+
+
+class UAPVisualizer:
+    def __init__(self, data=None):
+        pass  # Initialization can be added if needed
+
+    def analyze_and_predict(self, data, analyzers, col_names):
+        new_data = pd.DataFrame()
+        for i, (column, analyzer) in enumerate(zip(col_names, analyzers)):
+            new_data[f'Analyzer_{column}'] = analyzer.__dict__['cluster_terms']
+            print(f"Cluster terms extracted for {column}")
+
+        new_data = new_data.fillna('null').astype('category')
+        data_nums = new_data.apply(lambda x: x.cat.codes)
+
+        for col in data_nums.columns:
+            try:
+                categories = new_data[col].cat.categories
+                x_train, x_test, y_train, y_test = train_test_split(data_nums.drop(columns=[col]), data_nums[col], test_size=0.2, random_state=42)
+                bst, accuracy, preds = self.train_xgboost(x_train, y_train, x_test, y_test, len(categories))
+                self.plot_results(new_data, bst, x_test, y_test, preds, categories, accuracy, col)
+            except Exception as e:
+                print(f"Error processing {col}: {e}")
+
+    def train_xgboost(self, x_train, y_train, x_test, y_test, num_classes):
+        dtrain = xgb.DMatrix(x_train, label=y_train, enable_categorical=True)
+        dtest = xgb.DMatrix(x_test, label=y_test)
+
+        params = {'objective': 'multi:softmax', 'num_class': num_classes, 'max_depth': 6, 'eta': 0.3}
+        num_round = 100
+        bst = xgb.train(dtrain=dtrain, params=params, num_boost_round=num_round)
+        preds = bst.predict(dtest)
+        accuracy = accuracy_score(y_test, preds)
+
+        print(f"XGBoost trained with accuracy: {accuracy:.2f}")
+        return bst, accuracy, preds
+
+    def plot_results(self, new_data, bst, x_test, y_test, preds, categories, accuracy, col):
+        fig, axs = plt.subplots(1, 3, figsize=(25, 5))
+        fig.suptitle(f'{col.split(sep=".")[-1]} prediction', fontsize=35)
+
+        plot_importance(bst, ax=axs[0], importance_type='gain', show_values=False)
+        conf_matrix = confusion_matrix(y_test, preds)
+        sns.heatmap(conf_matrix, annot=True, fmt='g', cmap='Blues', xticklabels=categories, yticklabels=categories, ax=axs[1])
+        axs[1].set_title(f'Confusion Matrix\nAccuracy: {accuracy * 100:.2f}%')
+
+        sorted_features = sorted(bst.get_score(importance_type="gain").items(), key=lambda x: x[1], reverse=True)
+        most_important_feature = sorted_features[0][0]
+        contingency_table = pd.crosstab(new_data[col], new_data[most_important_feature])
+        chi2, p, dof, expected = chi2_contingency(contingency_table)
+        print(f"Chi-squared test for {col} and {most_important_feature}: p-value = {p}")
+        
+        sns.heatmap(contingency_table, annot=True, cmap='Greens', ax=axs[2])
+        axs[2].set_title(f'Contingency Table between {col.split(sep=".")[-1]} and {most_important_feature.split(sep=".")[-1]}\np-value = {p}')    
+
+        plt.tight_layout()
+        plt.savefig(f"{col}_{accuracy:.2f}_prediction_XGB.jpeg", dpi=300)
+        plt.show()
+
+    @staticmethod
+    def cramers_v(confusion_matrix):
+        chi2 = chi2_contingency(confusion_matrix)[0]
+        n = confusion_matrix.sum().sum()
+        phi2 = chi2 / n
+        r, k = confusion_matrix.shape
+        phi2corr = max(0, phi2 - ((k-1)*(r-1))/(n-1))
+        r_corr = r - ((r-1)**2)/(n-1)
+        k_corr = k - ((k-1)**2)/(n-1)
+        return np.sqrt(phi2corr / min((k_corr-1), (r_corr-1)))
+
+    def plot_cramers_v_heatmap(self, data, significance_level=0.05):
+        cramers_v_df = pd.DataFrame(index=data.columns, columns=data.columns, data=np.nan)
+        
+        for col1 in data.columns:
+            for col2 in data.columns:
+                if col1 != col2:
+                    confusion_matrix = pd.crosstab(data[col1], data[col2])
+                    chi2, p, dof, expected = chi2_contingency(confusion_matrix)
+                    if p < significance_level:
+                        cramers_v_df.at[col1, col2] = UAPVisualizer.cramers_v(confusion_matrix)
+        
+        plt.figure(figsize=(10, 8)),# facecolor="black")
+        mask = np.triu(np.ones_like(cramers_v_df, dtype=bool))
+        #sns.set_theme(style="dark", rc={"axes.facecolor": "black", "grid.color": "white", "xtick.color": "white", "ytick.color": "white", "axes.labelcolor": "white", "axes.titlecolor": "white"})
+        # ax = sns.heatmap(cramers_v_df, annot=True, fmt=".1f", linewidths=.5, linecolor='white', cmap='coolwarm', annot_kws={"color":"white"}, cbar=True, mask=mask, square=True)
+        # Customizing the color of the ticks and labels to white
+        # plt.xticks(color='white')
+        # plt.yticks(color='white')
+        sns.heatmap(cramers_v_df, annot=True, fmt=".2f", cmap='coolwarm', cbar=True, mask=mask, square=True)
+        plt.title(f"Heatmap of Cramér's V (p < {significance_level})")
+        plt.show()
+
+    
+    def plot_treemap(self, df, column, top_n=32):
+        # Get the value counts and the top N labels
+        value_counts = df[column].value_counts()
+        top_labels = value_counts.iloc[:top_n].index
+        
+
+        # Use np.where to replace all values not in the top N with 'Other'
+        revised_column = f'{column}_revised'
+        df[revised_column] = np.where(df[column].isin(top_labels), df[column], 'Other')
+
+        # Get the value counts including the 'Other' category
+        sizes = df[revised_column].value_counts().values
+        labels = df[revised_column].value_counts().index
+
+        # Get a gradient of colors
+        colors = list(mcolors.TABLEAU_COLORS.values())
+
+        # Get % of each category
+        percents = sizes / sizes.sum()
+
+        # Prepare labels with percentages
+        labels = [f'{label}\n {percent:.1%}' for label, percent in zip(labels, percents)]
+
+        # Plot the treemap
+        squarify.plot(sizes=sizes, label=labels, alpha=0.7, pad=True, color=colors, text_kwargs={'fontsize': 10})
+
+        ax = plt.gca()
+
+        # Iterate over text elements and rectangles (patches) in the axes for color adjustment
+        for text, rect in zip(ax.texts, ax.patches):
+            background_color = rect.get_facecolor()
+            r, g, b, _ = mcolors.to_rgba(background_color)
+            brightness = np.average([r, g, b])
+            text.set_color('white' if brightness < 0.5 else 'black')
+
+            # Adjust font size based on rectangle's area and wrap long text
+            coef = 0.8
+            font_size = np.sqrt(rect.get_width() * rect.get_height()) * coef
+            text.set_fontsize(font_size)
+            wrapped_text = textwrap.fill(text.get_text(), width=20)
+            text.set_text(wrapped_text)
+
+        plt.axis('off')
+        plt.gca().invert_yaxis()
+        plt.gcf().set_size_inches(20, 12)
+        plt.show()
+
+
+
+
+class UAPParser:
+    def __init__(self, api_key, model="gpt-3.5-turbo-0125", col=None, format_long=None):
+        os.environ['OPENAI_API_KEY'] = api_key
+        self.client = OpenAI()
+        self.model = model
+        self.responses = {}
+        self.col = None
+
+    def fetch_response(self, description, format_long):
+        INITIAL_WAIT_TIME = 5
+        MAX_WAIT_TIME = 600
+        MAX_RETRIES = 10
+
+        wait_time = INITIAL_WAIT_TIME
+        for attempt in range(MAX_RETRIES):
+            try:
+                response = self.client.chat.completions.create(
+                    model=self.model,
+                    response_format={"type": "json_object"},
+                    messages=[
+                        {"role": "system", "content": "You are a helpful assistant which is tasked to help parse data."},
+                        {"role": "user", "content": f'Input report: {description}\n\n Parse data following this json structure; leave missing data empty: {format_long}  Output:'}
+                    ]
+                )
+                return response
+            except HTTPError as e:
+                if 'TooManyRequests' in str(e):
+                    time.sleep(wait_time)
+                    wait_time = min(wait_time * 2, MAX_WAIT_TIME)  # Exponential backoff
+                else:
+                    raise
+            except Exception as e:
+                print(f"Unexpected error: {e}")
+                break
+
+        return None  # Return None if all retries fail
+
+    def process_descriptions(self, descriptions, format_long, max_workers=32):
+        with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
+            future_to_desc = {executor.submit(self.fetch_response, desc, format_long): desc for desc in descriptions}
+
+            for future in stqdm(concurrent.futures.as_completed(future_to_desc), total=len(descriptions)):
+                desc = future_to_desc[future]
+                try:
+                    response = future.result()
+                    response_text = response.choices[0].message.content if response else None
+                    if response_text:
+                        self.responses[desc] = response_text
+                except Exception as exc:
+                    print(f'Error occurred for description {desc}: {exc}')
+
+    def parse_responses(self):
+        parsed_responses = {}
+        not_parsed = 0
+        try:
+            for k, v in self.responses.items():
+                try:
+                    parsed_responses[k] = json.loads(v)
+                except:
+                    try:
+                        parsed_responses[k] = json.loads(v.replace("'", '"'))
+                    except:
+                        not_parsed += 1
+        except Exception as e:
+            print(f"Error parsing responses: {e}")
+ 
+        print(f"Number of unparsed responses: {not_parsed}")
+        print(f"Number of parsed responses: {len(parsed_responses)}")
+        return parsed_responses
+
+    def responses_to_df(self, col, parsed_responses):
+        parsed_df = pd.DataFrame(parsed_responses).T
+        if col is not None:
+            parsed_df2 = pd.json_normalize(parsed_df[col])
+            parsed_df2.index = parsed_df.index
+        else:
+            parsed_df2 = pd.json_normalize(parsed_df)
+            parsed_df2.index = parsed_df.index
+        return parsed_df2
+
+
+

uap_config.kgl

@@ -0,0 +1,239 @@
+{
+  "version": "v1",
+  "config": {
+    "visState": {
+      "filters": [],
+      "layers": [
+        {
+          "id": "cl6utnk",
+          "type": "point",
+          "config": {
+            "dataId": "uap_sightings",
+            "label": "Sightings",
+            "color": [
+              243,
+              251,
+              237
+            ],
+            "highlightColor": [
+              252,
+              242,
+              26,
+              255
+            ],
+            "columns": {
+              "lat": "latitude",
+              "lng": "longitude",
+              "altitude": null
+            },
+            "isVisible": true,
+            "visConfig": {
+              "radius": 9.7,
+              "fixedRadius": false,
+              "opacity": 0.8,
+              "outline": false,
+              "thickness": 2,
+              "strokeColor": null,
+              "colorRange": {
+                "name": "Global Warming",
+                "type": "sequential",
+                "category": "Uber",
+                "colors": [
+                  "#5A1846",
+                  "#900C3F",
+                  "#C70039",
+                  "#E3611C",
+                  "#F1920E",
+                  "#FFC300"
+                ]
+              },
+              "strokeColorRange": {
+                "name": "Global Warming",
+                "type": "sequential",
+                "category": "Uber",
+                "colors": [
+                  "#5A1846",
+                  "#900C3F",
+                  "#C70039",
+                  "#E3611C",
+                  "#F1920E",
+                  "#FFC300"
+                ]
+              },
+              "radiusRange": [
+                0,
+                50
+              ],
+              "filled": true
+            },
+            "hidden": false,
+            "textLabel": [
+              {
+                "field": null,
+                "color": [
+                  255,
+                  255,
+                  255
+                ],
+                "size": 18,
+                "offset": [
+                  0,
+                  0
+                ],
+                "anchor": "start",
+                "alignment": "center"
+              }
+            ]
+          },
+          "visualChannels": {
+            "colorField": null,
+            "colorScale": "quantile",
+            "strokeColorField": null,
+            "strokeColorScale": "quantile",
+            "sizeField": null,
+            "sizeScale": "linear"
+          }
+        },
+        {
+          "id": "bqpymx4",
+          "type": "heatmap",
+          "config": {
+            "dataId": "uap_sightings",
+            "label": "HeatMap",
+            "color": [
+              130,
+              154,
+              227
+            ],
+            "highlightColor": [
+              252,
+              242,
+              26,
+              255
+            ],
+            "columns": {
+              "lat": "latitude",
+              "lng": "longitude"
+            },
+            "isVisible": true,
+            "visConfig": {
+              "opacity": 0.8,
+              "colorRange": {
+                "name": "ColorBrewer BuPu-6",
+                "type": "sequential",
+                "category": "ColorBrewer",
+                "colors": [
+                  "#810f7c",
+                  "#8856a7",
+                  "#8c96c6",
+                  "#9ebcda",
+                  "#bfd3e6",
+                  "#edf8fb"
+                ],
+                "reversed": true
+              },
+              "radius": 42.2
+            },
+            "hidden": false,
+            "textLabel": [
+              {
+                "field": null,
+                "color": [
+                  255,
+                  255,
+                  255
+                ],
+                "size": 18,
+                "offset": [
+                  0,
+                  0
+                ],
+                "anchor": "start",
+                "alignment": "center"
+              }
+            ]
+          },
+          "visualChannels": {
+            "weightField": null,
+            "weightScale": "linear"
+          }
+        }
+      ],
+      "interactionConfig": {
+        "tooltip": {
+          "fieldsToShow": {
+            "040l4v2ys": [
+              {
+                "name": "0",
+                "format": null
+              },
+              {
+                "name": "labels",
+                "format": null
+              },
+              {
+                "name": "text",
+                "format": null
+              },
+              {
+                "name": "city",
+                "format": null
+              },
+              {
+                "name": "country",
+                "format": null
+              }
+            ]
+          },
+          "compareMode": false,
+          "compareType": "absolute",
+          "enabled": true
+        },
+        "brush": {
+          "size": 0.5,
+          "enabled": false
+        },
+        "geocoder": {
+          "enabled": false
+        },
+        "coordinate": {
+          "enabled": false
+        }
+      },
+      "layerBlending": "normal",
+      "splitMaps": [],
+      "animationConfig": {
+        "currentTime": null,
+        "speed": 1
+      }
+    },
+    "mapState": {
+      "bearing": 0,
+      "dragRotate": false,
+      "latitude": 42.965001236045275,
+      "longitude": -72.7338233315822,
+      "pitch": 0,
+      "zoom": 2.1144352988031674,
+      "isSplit": false
+    },
+    "mapStyle": {
+      "styleType": "dark",
+      "topLayerGroups": {},
+      "visibleLayerGroups": {
+        "label": true,
+        "road": true,
+        "border": false,
+        "building": true,
+        "water": true,
+        "land": true,
+        "3d building": false
+      },
+      "threeDBuildingColor": [
+        9.665468314072013,
+        17.18305478057247,
+        31.1442867897876
+      ],
+      "mapStyles": {}
+    }
+  }
+}