Update app.py

app.py

@@ -1,6 +1,7 @@
 import os
 import pandas as pd
 import numpy as np
+import gzip
 from sentence_transformers import SentenceTransformer
 from sklearn.metrics.pairwise import cosine_similarity
 import gradio as gr
@@ -18,6 +19,38 @@ question_embeddings = model.encode(df['Question'].tolist())
 # Together API setup
 client = Together(api_key=os.environ.get("TOGETHER_API_KEY"))
 
+def compressed_length(s):
+    return len(gzip.compress(s.encode('utf-8')))
+
+def ncd(x, y):
+    Cx = compressed_length(x)
+    Cy = compressed_length(y)
+    Cxy = compressed_length(x + " " + y)
+    return (Cxy - min(Cx, Cy)) / max(Cx, Cy)
+
+def normalize_scores(scores, reverse=False):
+    min_score = min(scores)
+    max_score = max(scores)
+    if reverse:
+        return [(max_score - x) / (max_score - min_score) for x in scores]
+    return [(x - min_score) / (max_score - min_score) for x in scores]
+
+def hybrid_retrieval(query, passages, embeddings, alpha=0.7, beta=0.3):
+    query_embedding = model.encode(query)
+    cosine_similarities = cosine_similarity([query_embedding], embeddings)[0]
+    
+    normalized_cosine_similarities = normalize_scores(cosine_similarities)
+    
+    ncd_values = [ncd(query, passage) for passage in passages]
+    normalized_ncd_values = normalize_scores(ncd_values, reverse=True)
+    
+    final_scores = [alpha * cos_sim + beta * ncd_sim 
+                    for cos_sim, ncd_sim in zip(normalized_cosine_similarities, normalized_ncd_values)]
+    
+    most_similar_index = np.argmax(final_scores)
+    
+    return most_similar_index, cosine_similarities[most_similar_index], ncd_values[most_similar_index], final_scores[most_similar_index]
+
 def llama_query(prompt, system_content):
     response = client.chat.completions.create(
         model="meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo",
@@ -75,26 +108,21 @@ def get_answer(user_question, threshold=0.01):
     else:
         english_question = user_question
 
-    user_embedding = model.encode(english_question)
-
-    similarities = cosine_similarity([user_embedding], question_embeddings)
-
-    max_similarity = np.max(similarities)
+    index, cosine_sim, ncd_value, final_score = hybrid_retrieval(english_question, df['Question'].tolist(), question_embeddings)
 
-    if max_similarity > threshold:
-        similar_question_idx = np.argmax(similarities)
-        retrieved_answer = df.iloc[similar_question_idx]['Answer']
+    if final_score > threshold:
+        retrieved_answer = df.iloc[index]['Answer']
         refined_answer = refine_answer(english_question, retrieved_answer)
 
         if language == 'swahili':
             refined_answer = translate_to_swahili(refined_answer)
 
-        return refined_answer, max_similarity
+        return refined_answer, final_score
     else:
         default_message = "The system couldn't find a sufficient answer to your question. Do you want to learn anything else about blood donation?"
         if language == 'swahili':
             default_message = translate_to_swahili(default_message)
-        return default_message, max_similarity
+        return default_message, final_score
 
 # Gradio app
 def gradio_app(user_question):