upload v1 of Dockerfile, rag, app, requirements and utils

Dockerfile

@@ -0,0 +1,24 @@
+FROM python:3.10-slim
+
+WORKDIR /app
+
+# System deps
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential git curl && \
+    rm -rf /var/lib/apt/lists/*
+
+# Python deps
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# App
+COPY . .
+
+# Cache tokenizer (optional)
+RUN python -c "from transformers import AutoTokenizer; AutoTokenizer.from_pretrained('mistralai/Mistral-7B-Instruct-v0.3')"
+
+ENV HF_HOME=/app/.cache/huggingface
+ENV TRANSFORMERS_CACHE=/app/.cache/huggingface
+
+EXPOSE 7860
+CMD ["python", "app.py"]

app.py

@@ -0,0 +1,191 @@
+import os, re, json, pandas as pd, gradio as gr, torch
+from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
+from typing import Optional
+
+from utils import (init_provenance_db, log_provenance, ensure_foundation_year_dir,
+                   download_pdf, find_best_report_url, DATA_DIR)
+from rag import add_pdf_to_index, get_retriever
+
+# ---------- Data & DB ----------
+FOUNDATIONS_CSV = "data/foundations.csv"
+foundations = pd.read_csv(FOUNDATIONS_CSV, dtype={"id":"int"})
+init_provenance_db()
+
+# ---------- LLM (local Mistral) ----------
+MODEL_ID = "mistralai/Mistral-7B-Instruct-v0.3"
+tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
+model = AutoModelForCausalLM.from_pretrained(MODEL_ID, torch_dtype=torch.float32)
+DEVICE = 0 if torch.cuda.is_available() else -1
+gen = pipeline("text-generation", model=model, tokenizer=tokenizer, device=DEVICE,
+               model_kwargs={"torch_dtype": torch.float32}, max_new_tokens=512, do_sample=False)
+
+# ---------- MCP tool: fetch_annual_report ----------
+def tool_fetch_annual_report(foundation_id: int, foundation_name: Optional[str] = None,
+                             year: Optional[int] = None, search_terms: Optional[str] = None,
+                             save_title: Optional[str] = None) -> dict:
+    if not foundation_name:
+        row = foundations[foundations["id"] == int(foundation_id)]
+        if row.empty:
+            return {"status":"error","message":f"Unknown foundation_id={foundation_id}"}
+        foundation_name = row.iloc[0]["name"]
+
+    try:
+        best = find_best_report_url(foundation_name, year, search_terms, serpapi_key=os.getenv("SERPAPI_KEY"))
+        if not best:
+            return {"status":"not_found","message":"No suitable report URL found."}
+        url = best.get("link")
+        title = save_title or best.get("title") or f"{foundation_name}-annual-report-{year or ''}".strip("-")
+
+        save_dir = ensure_foundation_year_dir(int(foundation_id), year)
+        saved_path = download_pdf(url, save_dir, preferred_name=f"{title}.pdf")
+
+        # Ingest into FAISS for RAG
+        add_pdf_to_index(saved_path, metadata={
+            "foundation_id": int(foundation_id),
+            "year": year,
+            "file_path": saved_path,
+            "source_url": url,
+            "title": title,
+            "doc_type": "annual_report"
+        })
+
+        # provenance
+        log_provenance(int(foundation_id), year, title, "annual_report", saved_path, url)
+
+        return {"status":"ok","url":url,"saved_path":saved_path,"message":f"Stored & indexed: {saved_path}"}
+    except Exception as e:
+        return {"status":"error","message":str(e)}
+
+# ---------- MCP extraction ----------
+def extract_function_call(text: str):
+    try:
+        data = json.loads(text.strip())
+        if isinstance(data, dict) and "function" in data and "parameters" in data:
+            return data["function"], data["parameters"]
+    except Exception:
+        pass
+    return None, None
+
+def system_prompt(context: str, user_question: str) -> str:
+    return f"""You are a Swiss philanthropy assistant with a tool (MCP-style).
+
+TOOL CALL FORMAT (STRICT JSON ONLY when calling a tool):
+{{
+  "function": "fetch_annual_report",
+  "parameters": {{
+    "foundation_id": <int>,
+    "foundation_name": "<string, optional>",
+    "year": <int, optional>,
+    "search_terms": "<string, optional>"
+  }}
+}}
+
+RULES:
+- If you need an annual report PDF URL, output ONLY the JSON tool call above.
+- Prefer precise PDF URLs; the tool will download + index the PDF automatically.
+- If you already have enough info to answer, reply normally (plain text), concise.
+
+Context:
+{context}
+
+User:
+{user_question}
+
+Your response (either JSON tool call or plain text):
+"""
+
+def llm(prompt: str) -> str:
+    out = gen(prompt)[0]["generated_text"]
+    # Return only the new segment after the prompt to avoid echo
+    return out[len(prompt):].strip() if out.startswith(prompt) else out.strip()
+
+def mcp_orchestrate(user_question: str):
+    context = ""
+    used_tool = False
+    for _ in range(3):
+        raw = llm(system_prompt(context, user_question))
+        fname, params = extract_function_call(raw)
+        if fname == "fetch_annual_report":
+            # Fill missing year by heuristic
+            if "year" not in params or not params["year"]:
+                m = re.search(r"\b(20\d{2}|19\d{2})\b", user_question)
+                if m: params["year"] = int(m.group(1))
+            res = tool_fetch_annual_report(
+                foundation_id=int(params["foundation_id"]),
+                foundation_name=params.get("foundation_name"),
+                year=params.get("year"),
+                search_terms=params.get("search_terms"),
+            )
+            context += f"\n[tool:fetch_annual_report -> {json.dumps(res, ensure_ascii=False)}]\n"
+            used_tool = True
+            continue
+        else:
+            return raw, used_tool
+    # Final pass to get a text response after tool
+    final = llm(system_prompt(context, user_question))
+    return final, used_tool
+
+# ---------- RAG answering ----------
+def rag_answer(question: str):
+    retriever = get_retriever(k=5)
+    # simple manual RAG: fetch docs, stuff into prompt
+    docs = retriever.get_relevant_documents(question)
+    sources = []
+    context = ""
+    for d in docs:
+        sources.append({
+            "page_content": d.page_content[:500],
+            "file_path": d.metadata.get("file_path"),
+            "page": d.metadata.get("page", "N/A"),
+            "year": d.metadata.get("year"),
+            "foundation_id": d.metadata.get("foundation_id")
+        })
+        context += f"\n[Source chunk]\n{d.page_content}\n"
+
+    prompt = f"""You are answering based ONLY on the context chunks below. If unsure, say you don't know.
+Context:
+{context}
+
+Question: {question}
+Answer concisely:"""
+    answer = llm(prompt)
+    return answer, sources
+
+# ---------- Gradio UI ----------
+def ask(user_input: str):
+    # 1) Let the model decide if it needs to call the fetch tool
+    model_reply, used_tool = mcp_orchestrate(user_input)
+
+    # 2) Always try a RAG answer (in case the user asked about content)
+    rag_resp, sources = rag_answer(user_input)
+
+    # Decision: if model_reply is a normal sentence (not JSON) and used_tool=False, show RAG answer primarily
+    # If used_tool=True, show model confirmation + RAG.
+    if used_tool and model_reply:
+        header = "✅ Tool used: report fetched/indexed.\n\n"
+        final = header + model_reply + "\n\n" + "— RAG answer —\n" + rag_resp
+    elif model_reply and not model_reply.strip().startswith("{"):
+        final = rag_resp  # prioritize grounded RAG
+    else:
+        final = rag_resp
+
+    # Pretty-print top sources
+    src_lines = []
+    for s in sources[:3]:
+        src_lines.append(f"- {s.get('file_path')} (page {s.get('page')}, year={s.get('year')}, id={s.get('foundation_id')})")
+    if src_lines:
+        final += "\n\nSources:\n" + "\n".join(src_lines)
+
+    return final
+
+with gr.Blocks() as demo:
+    gr.Markdown("## Swiss Philanthropy Assistant (Mistral + MCP/SerpAPI + RAG/FAISS)")
+    gr.Markdown("Ask to fetch a foundation’s annual report (by ID/name/year), then ask questions about its content. PDFs are downloaded, indexed, and queryable.")
+    inp = gr.Textbox(label="Your question", placeholder="e.g., Fetch the 2023 annual report for foundation ID 1, then summarize grants by theme.")
+    out = gr.Textbox(label="Assistant", lines=18)
+    btn = gr.Button("Ask")
+    btn.click(ask, inputs=inp, outputs=out)
+    inp.submit(ask, inputs=inp, outputs=out)
+
+if __name__ == "__main__":
+    demo.launch(server_name="0.0.0.0", server_port=7860)

rag.py

@@ -0,0 +1,47 @@
+from pathlib import Path
+from typing import List, Dict, Any, Optional
+from langchain_community.document_loaders import PyPDFLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_community.vectorstores import FAISS
+from langchain_community.embeddings import HuggingFaceEmbeddings
+
+INDEX_DIR = Path("data/vectorstore/faiss_index")
+INDEX_DIR.mkdir(parents=True, exist_ok=True)
+
+# Small + strong enough CPU embedding
+EMB_MODEL = "sentence-transformers/all-MiniLM-L6-v2"
+
+def load_embeddings():
+    return HuggingFaceEmbeddings(model_name=EMB_MODEL)
+
+def split_pdf(file_path: str):
+    loader = PyPDFLoader(file_path)
+    pages = loader.load()
+    splitter = RecursiveCharacterTextSplitter(chunk_size=1024, chunk_overlap=64)
+    return splitter.split_documents(pages)
+
+def _faiss_paths():
+    return str(INDEX_DIR / "index.faiss"), str(INDEX_DIR / "index.pkl")
+
+def load_or_create_faiss(emb):
+    faiss_path, pkl_path = _faiss_paths()
+    if Path(faiss_path).exists() and Path(pkl_path).exists():
+        return FAISS.load_local(INDEX_DIR, emb, allow_dangerous_deserialization=True)
+    # empty new index
+    return FAISS.from_texts([""], emb).delete(["0"]) or FAISS(embeddings=emb, index=None, docstore=None, index_to_docstore_id=None)
+
+def add_pdf_to_index(file_path: str, metadata: Optional[Dict[str, Any]] = None):
+    emb = load_embeddings()
+    vectordb = load_or_create_faiss(emb)
+    splits = split_pdf(file_path)
+    # attach metadata to each chunk
+    md = metadata or {}
+    for d in splits:
+        d.metadata.update(md)
+    vectordb.add_documents(splits)
+    vectordb.save_local(INDEX_DIR)
+
+def get_retriever(k: int = 4):
+    emb = load_embeddings()
+    vectordb = load_or_create_faiss(emb)
+    return vectordb.as_retriever(search_kwargs={"k": k})

requirements.txt

@@ -0,0 +1,11 @@
+gradio==4.44.1
+transformers>=4.42.0
+torch>=2.1.0
+pandas
+requests
+python-dateutil
+faiss-cpu
+pypdf
+langchain>=0.2.7
+langchain-community>=0.2.7
+sentence-transformers>=2.6.1

utils.py

@@ -0,0 +1,91 @@
+import os, re, sqlite3, datetime, requests
+from pathlib import Path
+from typing import Optional, List, Dict
+
+DATA_DIR = Path("data")
+PROV_DB = "provenance.db"
+
+# ---------- SQLite provenance ----------
+def init_provenance_db(db_path: str = PROV_DB):
+    conn = sqlite3.connect(db_path)
+    c = conn.cursor()
+    c.execute("""
+    CREATE TABLE IF NOT EXISTS retrieved_docs (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        foundation_id INTEGER NOT NULL,
+        year INTEGER,
+        title TEXT,
+        doc_type TEXT,
+        file_path TEXT,
+        source_url TEXT,
+        fetched_at TEXT DEFAULT CURRENT_TIMESTAMP
+    )""")
+    conn.commit(); conn.close()
+
+def log_provenance(foundation_id: int, year: Optional[int], title: str,
+                   doc_type: str, file_path: str, source_url: str,
+                   db_path: str = PROV_DB):
+    conn = sqlite3.connect(db_path); c = conn.cursor()
+    c.execute("""INSERT INTO retrieved_docs
+        (foundation_id, year, title, doc_type, file_path, source_url, fetched_at)
+        VALUES (?,?,?,?,?,?,?)""",
+        (foundation_id, year, title, doc_type, file_path, source_url,
+         datetime.datetime.now().isoformat()))
+    conn.commit(); conn.close()
+
+# ---------- Filesystem ----------
+def safe_filename(name: str) -> str:
+    name = re.sub(r"[^\w\-. ]+", "_", name)
+    return re.sub(r"\s+", "_", name).strip("_")
+
+def ensure_foundation_year_dir(fid: int, year: Optional[int]) -> Path:
+    base = DATA_DIR / f"{fid}_data"
+    if year: base = base / str(year)
+    base.mkdir(parents=True, exist_ok=True)
+    return base
+
+def download_pdf(url: str, save_dir: Path, preferred_name: Optional[str] = None) -> str:
+    filename = preferred_name or url.split("/")[-1].split("?")[0]
+    if not filename.lower().endswith(".pdf"):
+        filename += ".pdf"
+    filename = safe_filename(filename)
+    target = save_dir / filename
+    r = requests.get(url, stream=True, timeout=30); r.raise_for_status()
+    with open(target, "wb") as f:
+        for chunk in r.iter_content(8192):
+            if chunk: f.write(chunk)
+    return str(target)
+
+# ---------- SerpAPI search ----------
+def serpapi_search(query: str, num_results: int = 20, serpapi_key: Optional[str] = None) -> List[Dict]:
+    key = serpapi_key or os.getenv("SERPAPI_KEY")
+    if not key:
+        raise RuntimeError("SERPAPI_KEY not set (add it in HF Space Secrets).")
+    params = {"engine": "google", "q": query, "num": num_results, "api_key": key}
+    resp = requests.get("https://serpapi.com/search", params=params, timeout=20)
+    resp.raise_for_status()
+    return resp.json().get("organic_results", [])
+
+def _is_pdf_link(link: str) -> bool:
+    l = link.lower()
+    return l.endswith(".pdf") or (".pdf" in l)
+
+def score_candidate(item: Dict, foundation_name: str, year: Optional[int]) -> float:
+    title = (item.get("title") or "").lower()
+    link = (item.get("link") or "").lower()
+    score = 0.0
+    if any(k in title for k in ["annual", "report", "jahresbericht", "rapport", "rapport annuel"]): score += 2
+    if foundation_name.lower()[:10] in title or foundation_name.lower()[:10] in link: score += 1.5
+    if year and (str(year) in title or str(year) in link): score += 1.5
+    if _is_pdf_link(link): score += 1.0
+    return score
+
+def find_best_report_url(foundation_name: str, year: Optional[int], extra_terms: Optional[str], serpapi_key: Optional[str]) -> Optional[Dict]:
+    q = f'{foundation_name} annual report'
+    if year: q += f' {year}'
+    if extra_terms: q += f' {extra_terms}'
+    q += ' filetype:pdf site:org | site:ch | site:foundation | site:stiftung | site:fondation'
+    results = serpapi_search(q, num_results=20, serpapi_key=serpapi_key)
+    if not results: return None
+    ranked = sorted(results, key=lambda r: score_candidate(r, foundation_name, year), reverse=True)
+    return ranked[0]