url loading testing

app.py

@@ -2,6 +2,7 @@ import chainlit as cl
 from helper_functions import process_file, load_documents_from_url, add_to_qdrant
 import models
 import agents
+import asyncio
 
 @cl.on_chat_start
 async def on_chat_start():
@@ -20,38 +21,35 @@ def rename(orig_author: str):
 
 @cl.on_message
 async def main(message: cl.Message):
-    await cl.Message(
-        content=f"Received: {message.content}",
-    ).send()
     if message.content.startswith("http://") or message.content.startswith("https://"):
         message_type = "url"
     else:
         message_type = "question"
-    
-    await cl.Message(
-        content=f"message_type: {message_type}",
-    ).send()
 
     if message_type == "url":
-        # load the file
-        docs = load_documents_from_url(message.content)
-        await cl.Message(content="loaded docs").send()
-        splits = models.semanticChunker_tuned.split_documents(docs)
-        await cl.Message(content="split docs").send()
-        for i, doc in enumerate(splits):
-            doc.metadata["user_upload_source"] = f"source_{i}"
-        print(f"Processing {len(docs)} text chunks")
+        try:
+            # Run the document loading and splitting in a thread
+            docs = await asyncio.to_thread(load_documents_from_url, message.content)
+            await cl.Message(content="loaded docs").send()
 
-        # Add to the qdrant_store
-        qdrant_store.add_documents(
-            documents=splits
-        )
+            splits = await asyncio.to_thread(models.semanticChunker_tuned.split_documents, docs)
+            await cl.Message(content="split docs").send()
 
-        await cl.Message(f"Processing `{response.url}` done. You can now ask questions!").send()
-    
+            for i, doc in enumerate(splits):
+                doc.metadata["user_upload_source"] = f"source_{i}"
+            print(f"Processing {len(docs)} text chunks")
+
+            # Add to the qdrant_store asynchronously
+            await asyncio.to_thread(qdrant_store.add_documents, splits)
+
+            await cl.Message(f"Processing `{message.content}` done. You can now ask questions!").send()
+
+        except Exception as e:
+            await cl.Message(f"Error processing the document: {e}").send()
     else:
-        response = retrieval_augmented_qa_chain.invoke({"question": message.content})
-        await cl.Message(content=response.content).send()
+        # Handle the question as usual
+        response = await asyncio.to_thread(retrieval_augmented_qa_chain.invoke, {"question": message.content})
+        await cl.Message(content=response['content']).send()
 
     res = await ask_action()
     await handle_response(res)

requirements.txt

@@ -641,6 +641,7 @@ zipp==3.20.2
     #   -r requirements.in
     #   importlib-metadata
 unstructured
+asyncio
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools