Retry

app.py

@@ -1,131 +1,267 @@
 import os
-os.environ["CUDA_VISIBLE_DEVICES"] = ""  # Disable CUDA initialization
 os.environ["allow_dangerous_deserialization"] = "True"
 print(os.getcwd())
 embedding_path="/home/user/app/docs/_embeddings/index.faiss"
 print(f"Loading FAISS index from: {embedding_path}")
 if not os.path.exists(embedding_path):
     print("File not found!")
-
 HF_KEY=os.getenv('Gated_Repo')
 
 import spaces
-import sys
 import time
-import re
-import threading
-from typing import List, Dict
+from typing import final
+import asyncio
 
 import torch
 import gradio as gr
+import threading
+import re
 
-import pickle
+from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain_community.docstore import InMemoryDocstore
 from langchain_community.document_loaders import TextLoader
+from langchain.docstore.document import Document as LangchainDocument
 from langchain_community.llms.huggingface_pipeline import HuggingFacePipeline
+from langchain_core.indexing import index
+from langchain_core.vectorstores import VectorStore
+from llama_index.core.node_parser import TextSplitter
+from llama_index.legacy.vector_stores import FaissVectorStore
+from pycparser.ply.yacc import token
+from ragatouille import RAGPretrainedModel
+
+from langchain_text_splitters import MarkdownHeaderTextSplitter, CharacterTextSplitter
+from sentence_transformers import SentenceTransformer
+from sqlalchemy.testing.suite.test_reflection import metadata
+from sympy.solvers.diophantine.diophantine import length
+from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig, TextIteratorStreamer
+from transformers import pipeline
+
+#DEPR:from langchain.vectorstores import FAISS
+import faiss
 from langchain_community.vectorstores import FAISS
+#DEPR: from langchain_community.embeddings import HuggingFaceEmbeddings
 from langchain_huggingface import HuggingFaceEmbeddings
-from langchain.docstore.document import Document as LangchainDocument
-from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain_community.vectorstores.utils import DistanceStrategy
-from sentence_transformers import SentenceTransformer
-from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig, TextIteratorStreamer, pipeline, TextStreamer
 from huggingface_hub import login
 
+# Press Umschalt+F10 to execute it or replace it with your code.
+# Press Double Shift to search everywhere for classes, files, tool windows, actions, and settings.
+
 login(token=HF_KEY)
 
 vectorstore=None
 
-class CustomTextStreamer(TextStreamer):
-    def __init__(self, tokenizer):
-        super().__init__(tokenizer)
-
-    def put(self, token_ids):
-        text = self.tokenizer.decode(token_ids, skip_special_tokens=True)
-        print(text, end='', flush=True)  # Direktes Streaming auf der Konsole
-        yield text  # Yield für Gradio oder ähnliche Frameworks
-
 class BSIChatbot:
-    def __init__(self, model_paths: Dict[str, str], docs_path: str):
+    embedding_model = None
+    llmpipeline = None
+    llmtokenizer = None
+    vectorstore = None
+    rerankingModel = None
+    streamer = None
+    images = [None]
+
+    # model_paths = {
+    #    'llm_path': 'meta-llama/Llama-3.2-3B-Instruct',
+    #    'embed_model_path': 'intfloat/multilingual-e5-large-instruct',
+    #    'rerank_model_path': 'domci/ColBERTv2-mmarco-de-0.1'
+    # }
+
+    llm_path = "meta-llama/Llama-3.2-3B-Instruct"
+    word_and_embed_model_path = "intfloat/multilingual-e5-large-instruct"
+    docs = "/home/user/app/docs"
+    #docs = "H:\\Uni\\Master\\Masterarbeit\\Masterarbeit\\daten\\_parsed_embed_test"
+    rerankModelPath="domci/ColBERTv2-mmarco-de-0.1"
+    embedPath="/home/user/app/docs_embeddings"
+
+    def __init__(self):
         self.embedding_model = None
-        self.llmpipeline = None
-        self.llmtokenizer = None
-        #self.vectorstore = None
-        self.reranking_model = None
-        self.streamer = None
-        self.images = [None]
-
-        self.llm_path = model_paths['llm_path']
-        self.word_and_embed_model_path = model_paths['embed_model_path']
-        self.docs = docs_path
-        self.rerank_model_path = model_paths['rerank_model_path']
-
-    #@spaces.GPU
-    def initialize_embedding_model(self, rebuild_embeddings: bool):
-        raw_knowledge_base = []
+        #self.vectorstore: VectorStore = None
+
+    def initializeEmbeddingModel(self, new_embedding):
         global vectorstore
-        # Initialize embedding model
+        RAW_KNOWLEDGE_BASE = []
+
+        #Embedding, Vector generation and storing:
         self.embedding_model = HuggingFaceEmbeddings(
             model_name=self.word_and_embed_model_path,
             multi_process=True,
             model_kwargs={"device": "cuda"},
-            encode_kwargs={"normalize_embeddings": True},
+            encode_kwargs={"normalize_embeddings": True},  # Set `True` for cosine similarity
         )
 
-        if rebuild_embeddings:
-            # Load documents
-            for doc in os.listdir(self.docs):
-                file_path = os.path.join(self.docs, doc)
-                if doc.endswith(".md") or doc.endswith(".txt"):
-                    with open(file_path, 'r', encoding='utf-8' if doc.endswith(".md") else 'cp1252') as file:
-                        content = file.read()
-                        metadata = {"source": doc}
-                        raw_knowledge_base.append(LangchainDocument(page_content=content, metadata=metadata))
-
-            # Split documents into chunks
+        #index_cpu = faiss.IndexFlatL2(1024)
+        #res = faiss.StandardGpuResources()
+        #index_gpu = faiss.index_cpu_to_gpu(res, 0, index_cpu)
+        dirList = os.listdir(self.docs)
+        if (new_embedding==True):
+            for doc in dirList:
+                print(doc)
+                if (".md" in doc):
+                    ##doctxt = TextLoader(docs + "\\" + doc).load()
+                    file = open(self.docs + "\\" + doc, 'r', encoding='utf-8')
+                    doctxt = file.read()
+                    RAW_KNOWLEDGE_BASE.append(LangchainDocument(page_content=doctxt, metadata={"source": doc}))
+                    file.close()
+                if (".txt" in doc):
+                    file = open(self.docs + "\\" + doc, 'r', encoding='cp1252')
+                    doctxt = file.read()
+                    if doc.replace(".txt",".png") in dirList:
+                        RAW_KNOWLEDGE_BASE.append(LangchainDocument(page_content=doctxt, metadata={"source": doc.replace(".txt",".png")}))
+                    if doc.replace(".txt",".jpg") in dirList:
+                        RAW_KNOWLEDGE_BASE.append(LangchainDocument(page_content=doctxt, metadata={"source": doc.replace(".txt",".jpg")}))
+                    file.close()
+
+                    # RAW_KNOWLEDGE_BASE.append(txtLoader)
+                    # print(RAW_KNOWLEDGE_BASE)
+
+            # Chunking starts here
+
+            headers_to_split_on = [
+                ("#", "Header 1"),
+                ("##", "Header 2"),
+                ("###", "Header 3"),
+                ("####", "Header 4"),
+                ("#####", "Header 5"),
+            ]
+
+            markdown_splitter = MarkdownHeaderTextSplitter(
+                headers_to_split_on=headers_to_split_on,
+                strip_headers=True
+            )
+
             tokenizer = AutoTokenizer.from_pretrained(self.word_and_embed_model_path)
+
             text_splitter = RecursiveCharacterTextSplitter.from_huggingface_tokenizer(
                 tokenizer=tokenizer,
-                chunk_size=512,
-                chunk_overlap=0,
-                add_start_index=True,
-                strip_whitespace=True,
+                chunk_size=512,  # The maximum number of words in a chunk
+                chunk_overlap=0,  # The number of characters to overlap between chunks
+                add_start_index=True,  # If `True`, includes chunk's start index in metadata
+                strip_whitespace=True,  # If `True`, strips whitespace from the start and end of every document
             )
 
-            processed_docs = []
-            for doc in raw_knowledge_base:
-                chunks = text_splitter.split_documents([doc])
-                for chunk in chunks:
+            ##Was macht man mit start Index herausfinden und wie metadata adden
+            docs_processed = []
+            for doc in RAW_KNOWLEDGE_BASE:
+                print(f"Word-Length in doc:{len(doc.page_content.split())}")
+                doc_cache = markdown_splitter.split_text(doc.page_content)
+                # print(f"Word-Length in doc_cache after MarkdownSplitter:{len(doc_cache.split())}")
+                doc_cache = text_splitter.split_documents(doc_cache)
+                # print(f"Word-Length in doc_cache after text_splitter:{len(doc_cache.split())}")
+                for chunk in doc_cache:
                     chunk.metadata.update({"source": doc.metadata['source']})
-                processed_docs.extend(chunks)
-
-            # Create and save vector store
-            vectorstore = FAISS.from_documents(processed_docs, self.embedding_model, distance_strategy=DistanceStrategy.COSINE)
-            vectorstore.save_local(os.path.join(self.docs, "_embeddings"))
+                    print(f"Chunk_Debug len: {len(chunk.page_content.split())} and Chunk:{chunk}")
+                # DEBUG:
+                # print(f"doc_cache after Metadata added:{doc_cache}\n")
+                docs_processed += doc_cache
+
+            #final_docs = []
+            #for doc in docs_processed:
+            #   final_docs += text_splitter.split_documents([doc])
+
+            #docs_processed = final_docs
+
+            ##Ab hier alt:
+            # MARKDOWN_SEPARATORS = [
+            #    "\n\n",
+            #    "---"
+            #    "\n",
+            #    " ",
+            #    ""
+            # ]
+
+            #text_splitter = RecursiveCharacterTextSplitter(
+            #    chunk_size=512,  # The maximum number of characters in a chunk
+            #    chunk_overlap=100,  # The number of characters to overlap between chunks
+            #    add_start_index=True,  # If `True`, includes chunk's start index in metadata
+            #    strip_whitespace=True,  # If `True`, strips whitespace from the start and end of every document
+            #    separators=MARKDOWN_SEPARATORS,
+            #)
+
+            #docs_processed = []
+            #for doc in RAW_KNOWLEDGE_BASE:
+            #    docs_processed += text_splitter.split_documents([doc])
+
+            print(f"Docs processed:{len(docs_processed)}")
+            # Max_Sequence_Length of e5 large instr = 512 Tokens
+
+
+            # Make sure the maximum length is below embedding size
+            lengths = [len(s.page_content) for s in docs_processed]
+            print(max(lengths))
+
+            #for l in docs_processed:
+            #    print(f"Char-Length:{len(l.page_content.split())}")
+            #    print(f"Tokenizer Length: {len(tokenizer.tokenize(l.page_content))}")
+
+            #if (max(lengths) > SentenceTransformer(self.word_and_embed_model_path).max_seq_length):
+            #    print(
+            #        f'Error: Fit chunking size into embedding model.. Chunk{max(lengths)} is bigger than {SentenceTransformer(self.word_and_embed_model_path).Max_Sequence_Length}')
+
+            start = time.time()
+            #docstore = InMemoryDocstore({str(i): doc for i, doc in enumerate(docs_processed)})
+            #index_to_docstore_id = {i: str(i) for i in range(len(docs_processed))}
+            vectorstore = FAISS.from_documents(docs_processed, self.embedding_model, distance_strategy=DistanceStrategy.COSINE)
+            #self.vectorstore = FAISS(
+            #    embedding_function=self.embedding_model,
+            #    index=index_gpu,
+            #    distance_strategy=DistanceStrategy.COSINE,
+            #    docstore=docstore,
+            #    index_to_docstore_id=index_to_docstore_id
+            #)
+            #self.vectorstore.from_documents(docs_processed, self.embedding_model)
+            #index_cpu = faiss.index_gpu_to_cpu(self.vectorstore.index)
+            #self.vectorstore.index = index_cpu
+            vectorstore.save_local(self.embedPath)
+            #self.vectorstore.index = index_gpu
+            end = time.time()
+            print("Saving Embeddings took", end-start, "seconds!")
         else:
-            # Load existing vector store
-            vectorstore = FAISS.load_local(os.path.join(self.docs, "_embeddings"), self.embedding_model, allow_dangerous_deserialization=True)
-            print("DBG: Vectorstore Status Initialization:", vectorstore)
+            start = time.time()
+            vectorstore = FAISS.load_local(self.embedPath, self.embedding_model, allow_dangerous_deserialization=True)
+            #self.vectorstore.index = index_gpu
+            end = time.time()
 
-    #@spaces.GPU
-    def retrieve_similar_embedding(self, query: str):
+            print("Loading Embeddings took", end - start, "seconds!")
+
+    def retrieveSimiliarEmbedding(self, query):
         global vectorstore
-        #lazy load
-        #if (self.vectorstore == None):
-        #   self.vectorstore = FAISS.load_local(os.path.join(self.docs, "_embeddings"), self.embedding_model,
-        #                                        allow_dangerous_deserialization=True)
-        print("DBG: Vectorstore Status retriever:", vectorstore)
+        print("Retrieving Embeddings...")
+        start = time.time()
         query = f"Instruct: Given a search query, retrieve the relevant passages that answer the query\nQuery:{query}"
-        return vectorstore.similarity_search(query=query, k=20)
 
-    #@spaces.GPU
-    def initialize_llm(self):
-        bnb_config = BitsAndBytesConfig(load_in_8bit=True)
-        llm = AutoModelForCausalLM.from_pretrained(self.llm_path, quantization_config=bnb_config)
-        #llm = AutoModelForCausalLM.from_pretrained(self.llm_path)
+        #self.vectorstore.
+        #retrieved_chunks = self.vectorstore.similarity_search(query=query, k=20)
+        retrieved_chunks = vectorstore.similarity_search(query=query, k=20)
+        #finalchunks = []
+        #for chunk in retrieved_chunks:
+        #    if "---" not in chunk.page_content:
+        #        finalchunks.append(chunk)
+        #retrieved_chunks = finalchunks
+        end = time.time()
+        print("Retrieving Chunks with similiar embeddings took", end - start, "seconds!")
+        #print("\n==================================Top document==================================")
+        #print(retrieved_chunks[0].page_content)
+        #print(retrieved_chunks[1].page_content)
+        #print(retrieved_chunks[2].page_content)
+        #print("==================================Metadata==================================")
+        #print(retrieved_chunks[0].metadata)
+        #print(retrieved_chunks[1].metadata)
+        #print(retrieved_chunks[2].metadata)
+        print(f"printing first chunk to see whats inside: {retrieved_chunks[0]}")
+        return retrieved_chunks
+
+    def initializeLLM(self):
+        bnb_config = BitsAndBytesConfig(
+            load_in_8bit=True,
+            #bnb_8bit_use_double_quant=True,
+            #bnb_8bit_quant_type="nf4",
+            #bnb_8bit_compute_dtype=torch.bfloat16,
+        )
+        llm = AutoModelForCausalLM.from_pretrained(
+            self.llm_path, quantization_config=bnb_config
+        )
         self.llmtokenizer = AutoTokenizer.from_pretrained(self.llm_path)
-        #self.streamer = TextIteratorStreamer(self.llmtokenizer, skip_prompt=True)
-        self.streamer = CustomTextStreamer(self.llmtokenizer)
+        self.streamer=TextIteratorStreamer(self.llmtokenizer, skip_prompt=True)
         self.llmpipeline = pipeline(
             model=llm,
             tokenizer=self.llmtokenizer,
@@ -134,84 +270,213 @@ class BSIChatbot:
             temperature=0.7,
             repetition_penalty=1.1,
             return_full_text=False,
-            #streamer=self.streamer,
+            streamer=self.streamer,
             max_new_tokens=500,
         )
 
-    @spaces.GPU
-    def rag_prompt(self, query: str, rerank: bool, history: List[Dict]):
-        retrieved_chunks = self.retrieve_similar_embedding(query)
-        retrieved_texts = [f"{chunk.metadata['source']}:\n{chunk.page_content}" for chunk in retrieved_chunks]
-
-        context = "\n".join(retrieved_texts)
-        history_text = "\n".join([h['content'] for h in history])
-        final_prompt = f"""Context:
-        {context}
-        ---
-        History:
-        {history_text}
-        ---
-        Question: {query}"""
-
-        input_ids = self.llmtokenizer(final_prompt, return_tensors="pt").to(self.llmpipeline.model.device)
-        streamer=self.streamer
-
-        self.llmpipeline.model.generate(input_ids=input_ids, streamer=streamer, max_new_tokens=500, temperature=0.7, repetition_penalty=1.1)
-
-        for text in streamer:
-            yield text
-
-        #input_ids = self.llmtokenizer(final_prompt, return_tensors="pt").to(self.llmpipeline.model.device)
-        #streamer = TextIteratorStreamer(self.llmtokenizer, skip_prompt=True)
-        #generate_kwargs = {
-        #    "input_ids": input_ids,
-        #    "streamer": streamer,
-        #    "max_new_tokens": 500,
-        #    "temperature": 0.7,
-        #    "repetition_penalty": 1.1,
-        #    "do_sample": True,
-        #}
-
-        #generation_thread = threading.Thread(target=self.llmpipeline.model.generate, kwargs=generate_kwargs)
-        #generation_thread.start()
-
-        #for text in streamer:
-            #yield text
-
-    def launch_interface(self):
-        with gr.Blocks() as demo:
-            chatbot = gr.Chatbot(type="messages")
-            msg = gr.Textbox()
-            clear = gr.Button("Clear")
-            reset = gr.Button("Reset")
-
-            def user_input(user_message, history):
-                return "", history + [{"role": "user", "content": user_message}]
-
-            def bot_response(history):
-                response = self.rag_prompt(history[-1]['content'], True, history)
-                history.append({"role": "assistant", "content": ""})
-
-                for token in response:
-                    history[-1]['content'] += token
-                    #print("Current Response Token Serializable?", pickle.dumps(token))
-                    yield history
-
-            msg.submit(user_input, [msg, chatbot], [msg, chatbot]).then(bot_response, chatbot, chatbot)
-            clear.click(lambda: None, None, chatbot)
-            reset.click(lambda: [], outputs=chatbot)
-
-        demo.launch()
+    def queryLLM(self,query):
+        #resp = self.llmpipeline(chat) Fixen
+        return(self.llmpipeline(query)[0]["generated_text"])
+
+    def initializeRerankingModel(self):
+        self.rerankingModel = RAGPretrainedModel.from_pretrained(self.rerankModelPath)
+
+
+    def ragPrompt(self, query, rerankingStep, history):
+        prompt_in_chat_format = [
+            {
+                "role": "system",
+                "content": """You are an helpful Chatbot for the BSI IT-Grundschutz. Using the information contained in the context,
+                give a comprehensive answer to the question.
+                Respond only to the question asked, response should be concise and relevant but also give some context to the question. 
+                Provide the source document when relevant for the understanding.
+                If the answer cannot be deduced from the context, do not give an answer.""",
+            },
+            {
+                "role": "user",
+                "content": """Context:
+                {context}
+                ---
+                Chat-History:
+                {history}
+                ---
+                Now here is the question you need to answer.
+
+                Question: {question}""",
+            },
+        ]
+        RAG_PROMPT_TEMPLATE = self.llmtokenizer.apply_chat_template(
+            prompt_in_chat_format, tokenize=False, add_generation_prompt=True
+        )
+        retrieved_chunks = self.retrieveSimiliarEmbedding(query)
+        retrieved_chunks_text = []
+        #TODO Irgendwas stimmt hier mit den Listen nicht
+        for chunk in retrieved_chunks:
+            #TODO Hier noch was smarteres Überlegen für alle Header
+            if "Header 1" in chunk.metadata.keys():
+                retrieved_chunks_text.append(f"The Document is: '{chunk.metadata["source"]}'\nHeader of the Section is: '{chunk.metadata["Header 1"]}' and Content of it:{chunk.page_content}")
+            else:
+                retrieved_chunks_text.append(
+                    f"The Document is: '{chunk.metadata["source"]}'\nImage Description is: ':{chunk.page_content}")
+        i=1
+        for chunk in retrieved_chunks_text:
+            print(f"Retrieved Chunk number {i}:\n{chunk}")
+            i=i+1
+
+        if rerankingStep==True:
+            if self.rerankingModel == None:
+                print ("initializing Reranker-Model..")
+                self.initializeRerankingModel()
+            print("Starting Reranking Chunks...")
+            self.rerankingModel
+            retrieved_chunks_text=self.rerankingModel.rerank(query, retrieved_chunks_text,k=5)
+            retrieved_chunks_text=[chunk["content"] for chunk in retrieved_chunks_text]
+
+            i = 1
+            for chunk in retrieved_chunks_text:
+                print(f"Reranked Chunk number {i}:\n{chunk}")
+                i = i + 1
+
+        context = "\nExtracted documents:\n"
+        context += "".join([doc for i, doc in enumerate(retrieved_chunks_text)])
+        #Alles außer letzte Useranfrage
+        final_prompt = RAG_PROMPT_TEMPLATE.format(
+            question=query, context=context, history=history[:-1]
+        )
+
+        print(f"Query:\n{final_prompt}")
+        pattern = r"Filename:(.*?);"
+        match = re.findall(pattern, final_prompt)
+        self.images=match
+
+        #queryModel = HuggingFacePipeline(pipeline = self.llmpipeline)
+        generation_thread = threading.Thread(target=self.llmpipeline, args=(final_prompt,))
+        generation_thread.start()
+
+        return self.streamer
+
+        #answer=self.queryLLM(final_prompt)
+        #answer = self.llmpipeline(final_prompt)
+        #for token in answer:
+        #    print (token["generated_text"])
+        #    yield token["generated_text"]
+        # gen = queryModel.stream(final_prompt)
+
+
+        #return gen
+
+        #print (f"Answer:\n{answer}")
+
+    def returnImages(self):
+        imageList = []
+        for image in self.images:
+            imageList.append(f"{self.docs}\\{image}")
+        return imageList
+
+    def launchGr(self):
+        gr.Interface.from_pipeline(self.llmpipeline).launch()
+
+
 
 if __name__ == '__main__':
-    model_paths = {
-        'llm_path': 'meta-llama/Llama-3.2-3B-Instruct',
-        'embed_model_path': 'intfloat/multilingual-e5-large-instruct',
-        'rerank_model_path': 'domci/ColBERTv2-mmarco-de-0.1'
-    }
-    docs_path = '/home/user/app/docs'
-
-    bot = BSIChatbot(model_paths, docs_path)
-    bot.initialize_embedding_model(rebuild_embeddings=False)
-    bot.initialize_llm()
-    bot.launch_interface()
+    #RAW_KNOWLEDGE_BASE = []
+    #RAW_KNOWLEDGE_BASE.append(LangchainDocument(page_content="1Text", metadata={"source": "bb"}))
+    #RAW_KNOWLEDGE_BASE.append(LangchainDocument(page_content="2Text", metadata={"source": "aa"}))
+    #RAW_KNOWLEDGE_BASE[0].metadata.update({"NeuerKey":"White"})
+    #print(RAW_KNOWLEDGE_BASE)
+    #time.sleep(10)
+
+    #{doc.page_content} [{doc.metadata}] => aktuellen Header in jeden Chunk embedden; Doc.Metadata retrieven
+
+    renewEmbeddings = False
+    reranking = True
+    bot = BSIChatbot()
+    bot.initializeEmbeddingModel(renewEmbeddings)
+    if reranking == True:
+        bot.initializeRerankingModel()
+    #TODO: DEBUG:
+    #bot.retrieveSimiliarEmbedding("Was ist der IT-Grundschutz?")
+    #TODO: DEBUG:
+    #time.sleep(10)
+    bot.initializeLLM()
+    #bot.retrieveSimiliarEmbedding("Welche Typen von Anforderungen gibt es im IT-Grundschutz?")
+
+    #bot.queryLLM("Welche Typen von Anforderungen gibt es im IT-Grundschutz?")
+
+    #bot.ragPrompt("""
+    #Welche Informationen beinhaltet die IT-Grundschutz-Methodik (BSI-Standard 200-2)? Wähle aus den folgenden Antwortmöglichkeiten (mehrere können richtig sein!):
+    #A: besonders schutzwürdigen Komponenten,
+    #B: methodische Hilfestellungen zur schrittweisen Einführung eines ISMS,
+    #C: wie die Informationssicherheit im laufenden Betrieb aufrechterhalten und kontinuierlich verbessert werden kann,
+    #D: effiziente Verfahren, um die allgemeinen Anforderungen des BSI-Standards 200-1 zu konkretisieren
+    #""", True)
+
+    #bot.launchGr()
+
+    with gr.Blocks() as demo:
+        with gr.Row() as row:
+                with gr.Column(scale=3):
+                    chatbot = gr.Chatbot(type="messages")
+                    msg = gr.Textbox()
+                    clear = gr.Button("Clear")
+                    reset = gr.Button("Reset")
+                with gr.Column(scale=1):  # Bildergalerie
+                    gallery = gr.Gallery(label="Bildergalerie",elem_id="gallery")
+
+        def user(user_message, history: list):
+            return "", history + [{"role": "user", "content": user_message}]
+
+
+        def returnImages():
+            # Hier holen wir uns die Bildpfade und wandeln sie in gr.Image-Objekte um
+            image_paths = bot.returnImages()
+            print(f"returning images: {image_paths}")
+            return image_paths
+
+        def gradiobot(history: list):
+            start = time.time()
+            print(f"ragQuery hist -1:{history[-1].get("content")}")
+            print(f"ragQuery hist 0:{history[0].get("content")}")
+            print(f"fullHistory: {history}" )
+            bot_response = bot.ragPrompt(history[-1].get("content"), reranking, history)
+            history.append({"role": "assistant", "content": ""})
+
+            image_gallery = returnImages()
+
+            for token in bot_response:
+                if "eot_id" in token:
+                    token = token.replace("<|eot_id|>","")
+                if token.startswith("-"):
+                    token = f"\n{token}"
+                if re.match(r"^[1-9]\.",token):
+                    token = f"\n{token}"
+
+                history[-1]['content'] += token
+                yield history, image_gallery
+            end = time.time()
+            print("End2End Query took", end - start, "seconds!")
+
+        def resetHistory():
+            return []
+
+        msg.submit(user, [msg, chatbot], [msg, chatbot], queue=False).then(
+            gradiobot, inputs=[chatbot], outputs=[chatbot, gallery]
+        )
+
+
+        clear.click(lambda: None, None, chatbot, queue=False)
+        reset.click(resetHistory, outputs=chatbot, queue=False)
+    demo.css = """
+        #gallery {
+            display: grid;
+            grid-template-columns: repeat(2, 1fr);
+            gap: 10px;
+            height: 400px;
+            overflow: auto;
+        }
+    """
+    demo.launch(allowed_paths=["/home/user/app/docs"])
+
+    #Answer: B, C und D => Korrekt!
+
+# See PyCharm help at https://www.jetbrains.com/help/pycharm/