Update rag_pipeline.py

rag_pipeline.py

@@ -1,70 +1,60 @@
-# rag_pipeline.py (Debug-Version mit Indexprüfung & Logging)
-
 import os
 import pickle
-import numpy as np
+import requests
 import faiss
+import numpy as np
 from sentence_transformers import SentenceTransformer
-from langchain.text_splitter import RecursiveCharacterTextSplitter, SentenceTransformersTokenTextSplitter
-import umap.umap_ as umap
-from dotenv import load_dotenv
-from groq import Groq
-from openai import OpenAI
-import tqdm
-
-print("🚀 RAG-App gestartet")
-
-# === Load environment variables (in HF Spaces über Secrets verfügbar) ===
-openai_api_key = os.getenv("OPENAI_API_KEY")
-groq_api_key = os.getenv("GROQ_API_KEY")
 
-groq_client = Groq(api_key=groq_api_key) if groq_api_key else None
-openai_client = OpenAI(api_key=openai_api_key) if openai_api_key else None
-
-# === Load SentenceTransformer model ===
-print("📦 Lade SentenceTransformer Modell...")
+# === Modell laden ===
 model = SentenceTransformer("Sahajtomar/German-semantic")
-print("✅ Modell geladen")
 
-# === Lade FAISS-Index und Chunk-Mapping ===
-try:
-    print("📂 Lade FAISS-Index...")
-    if not os.path.exists("faiss/faiss_index.index"):
-        raise FileNotFoundError("❌ faiss_index.index fehlt!")
+# === Google Drive Direktlinks
+url_index = "https://drive.google.com/uc?export=download&id=1QBg4vjitJ2xHEyp3Ae8TWJHwEHjbwgOO"
+url_chunks = "https://drive.google.com/uc?export=download&id=1nsrAm_ozsK4GlmMui9yqZBjmgUfqU2qa"
 
-    if not os.path.exists("faiss/chunks_mapping.pkl"):
-        raise FileNotFoundError("❌ chunks_mapping.pkl fehlt!")
+# === Lokale Pfade
+local_index = "faiss_index.index"
+local_chunks = "chunks_mapping.pkl"
 
-    index = faiss.read_index("faiss/faiss_index.index")
-    with open("faiss/chunks_mapping.pkl", "rb") as f:
-        token_split_texts = pickle.load(f)
+# === Download nur bei Bedarf
+def download_if_missing(url, local_path):
+    if not os.path.exists(local_path):
+        print(f"⬇️ Lade {local_path} von Google Drive...")
+        r = requests.get(url)
+        if r.status_code == 200:
+            with open(local_path, "wb") as f:
+                f.write(r.content)
+            print(f"✅ Heruntergeladen: {local_path}")
+        else:
+            raise Exception(f"❌ Download fehlgeschlagen für {local_path}")
 
-    chunk_embeddings = model.encode(token_split_texts, convert_to_numpy=True)
-    print("✅ FAISS & Embeddings geladen")
+download_if_missing(url_index, local_index)
+download_if_missing(url_chunks, local_chunks)
 
-    # UMAP initialisieren
-    umap_transform = umap.UMAP(random_state=0, transform_seed=0).fit(chunk_embeddings)
-    print("✅ UMAP fit abgeschlossen")
+# === Dateien laden
+print("📂 Lade FAISS Index und Chunks...")
+index = faiss.read_index(local_index)
 
-except Exception as e:
-    print(f"❌ Fehler beim Laden von FAISS oder Chunks: {e}")
-    index = None
-    token_split_texts = []
-    chunk_embeddings = None
-    umap_transform = None
+with open(local_chunks, "rb") as f:
+    token_split_texts = pickle.load(f)
 
-def project_embeddings(embeddings, umap_transform):
-    umap_embeddings = np.empty((len(embeddings), 2))
-    for i, embedding in enumerate(tqdm.tqdm(embeddings, desc="Projecting Embeddings")):
-        umap_embeddings[i] = umap_transform.transform([embedding])
-    return umap_embeddings
+chunk_embeddings = model.encode(token_split_texts, convert_to_numpy=True)
 
 def retrieve(query, k=5):
-    if index is None or chunk_embeddings is None:
-        return ["Kein Index verfügbar."], [], []
     query_embedding = model.encode([query], convert_to_numpy=True)
     distances, indices = index.search(query_embedding, k)
     retrieved_texts = [token_split_texts[i] for i in indices[0]]
+    return retrieved_texts
+
+def build_prompt(query, texts):
+    context = "\n\n".join(texts)
+    return f"Beantworte die folgende Frage basierend auf dem Kontext:\n\nKontext:\n{context}\n\nFrage:\n{query}"
+
+def run_qa_pipeline(query, k=5):
+    retrieved = retrieve(query, k)
+    prompt = build_prompt(query, retrieved)
+    return f"🔍 Kontext gefunden:\n\n{prompt}\n\n(Füge hier optional deine LLM-Antwort ein)"    distances, indices = index.search(query_embedding, k)
+    retrieved_texts = [token_split_texts[i] for i in indices[0]]
     retrieved_embeddings = np.array([chunk_embeddings[i] for i in indices[0]])
     return retrieved_texts, retrieved_embeddings, distances[0]