added EvalDataset Generation

app.py

@@ -684,7 +684,7 @@ class BSIChatbot:
 
     def generateEvalDataset(self):
         filepath = "/home/user/app/docs/_eval/BSI_Lektion_Ground_Truth.CSV"
-        with open(filepath, mode='r') as file:
+        with open(filepath, mode='r', encoding="latin1", errors="replace") as file:
             # Create a CSV reader with DictReader
             csv_reader = csv.DictReader(file, delimiter="|")
 
@@ -698,15 +698,19 @@ class BSIChatbot:
 
         # Print the list of dictionaries
         for data in data_list:
-            print(data)
+            print("starting to generate evaldataset..")
             data["Context"] = None
             data["Answer"] = None
             print(data)
-
-        for data in data_list[:3]:
+        i=1
+        #for data in data_list[:3]:
+        for data in data_list:
+            print("Eval Entry no:")
+            print(i)
             #def ragPromptNew(self, query, rerankingStep, history, stepBackPrompt)
             data["Answer"],data["Context"]  = ragPromptNew(data["Frage"],True,None,True)
             print(data)
+            i=i+1
 
         with open('/home/user/app/docs/_eval/eval_dataset.json', 'w') as fout:
             json.dump(data_list, fout)