implement data downloader script

.gitignore

@@ -5,6 +5,7 @@ __pycache__/
 
 # data 
 data
+demo 
 
 # C extensions
 *.so

app.py

@@ -1,12 +1,18 @@
 from src.data_processor import DataProcessor
 from src.embedding import EmbeddingManager
 from src.vectorstore import VectorStore
+from src.download_data import download_pmc_docs
 
 if __name__ == '__main__':
-    dp = DataProcessor()
-    chunks, document = dp.build()
-    embd = EmbeddingManager()
-    chunks_embedding = embd.embed_texts(chunks)
-    vectorstore = VectorStore()
-    vectorstore.add_documents(chunks, chunks_embedding)
-    retriver = vectorstore.get_retriever()
+    flag = True # download_pmc_docs()
+    if flag:
+        dp = DataProcessor()
+        chunks, document = dp.build()
+        chunks_list = [c.page_content for c in chunks]
+        embd = EmbeddingManager()
+        embd_model = embd.get_model()
+        chunks_embedding = embd.embed_texts(chunks_list)
+        vectorstore = VectorStore()
+        vectorstore.add_documents(chunks, chunks_embedding)
+        retriver = vectorstore.get_retriever(embd_model)
+    

src/download_data.py

@@ -0,0 +1,45 @@
+import os 
+
+from src.constant import BASE_DIR
+
+import boto3
+from botocore import UNSIGNED
+from botocore.client import Config
+
+
+
+TARGET_DIR = os.path.join(BASE_DIR, "data", "demo")
+
+def download_pmc_docs(
+        bucket="pmc-oa-opendata",
+        prefix="oa_comm/txt/all",
+        target_dir=TARGET_DIR,
+        limit=10
+):
+    os.makedirs(target_dir, exist_ok=True)
+
+    s3 = boto3.client("s3", config=Config(signature_version=UNSIGNED))
+    paginator = s3.get_paginator('list_objects_v2')
+
+    downloaded = 0
+
+    for page in paginator.paginate(Bucket=bucket, Prefix=prefix):
+        for obj in page.get("Contents", []):
+            key = obj["Key"]
+            if not key.endswith(".txt"):
+                continue
+            filename = os.path.basename(key)
+            local_path = os.path.join(target_dir, filename)
+            
+            if not os.path.exists(local_path):
+                s3.download_file(bucket, key, local_path)
+                downloaded += 1
+
+            if downloaded >= limit:
+                print(f"✅ Reached limit of {limit} documents.")
+                return
+        
+    print(f"✅ Finished. Total downloaded: {downloaded}")
+
+    return True 
+ 

src/llm.py

@@ -0,0 +1,22 @@
+import os 
+from langchain_google_genai import ChatGoogleGenerativeAI
+from langchain.chains import RetrievalQA
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+class LLM:
+    def __init__(self, retriever):
+        self.llm = ChatGoogleGenerativeAI(model="gemini-2.5-flash",
+                                           google_api_key=os.getenv("GOOGLE_API_KEY"))
+        self.qa = RetrievalQA.from_chain_type(
+            llm=self.llm,
+            retriever=retriever,
+            chain_type="stuff",
+            return_source_documents=True
+        )
+
+    def invoke(self, query: str):
+        result = self.qa.invoke({"query": query})        
+        return result

src/vectorstore.py

@@ -8,7 +8,7 @@ from langchain.vectorstores import Chroma
 from langchain.schema import Document
 from uuid import uuid4
 
-DATA_DIR = os.path.join(BASE_DIR, "db")
+DATA_DIR = os.path.join(BASE_DIR, "data", "db")
 
 
 class VectorStore: