Update app.py

app.py

@@ -37,7 +37,17 @@ model_name = "ggml-model-q4_1.gguf"
 
 snapshot_download(repo_id=repo_name, local_dir=".", allow_patterns=model_name)
 
-
+model = Llama(
+        model_path=model_name,
+        n_ctx=2000,
+        n_parts=1,
+        #n_batch=100,
+        logits_all=True,
+        #n_threads=12,
+        verbose=True,
+        n_gpu_layers=35,
+        n_gqa=8       #must be set for 70b models
+    )
 
 
 def get_message_tokens(model, role, content):
@@ -107,7 +117,7 @@ def generate_unknown_response():
 def generate_search_request():
     global stop_generation
     stop_generation = False
-
+    model.reset()
     
     
     data = request.get_json()
@@ -125,17 +135,7 @@ def generate_search_request():
     top_k = 20
     return_full_text = parameters.get("return_full_text", False)
 
-    model = Llama(
-        model_path=model_name,
-        n_ctx=2000,
-        n_parts=1,
-        #n_batch=100,
-        logits_all=True,
-        #n_threads=12,
-        verbose=True,
-        n_gpu_layers=30,
-        n_gqa=8       #must be set for 70b models
-    )
+    
     
     tokens = get_system_tokens_for_preprompt(model, preprompt)
     tokens.append(LINEBREAK_TOKEN)        
@@ -157,7 +157,7 @@ def generate_search_request():
 def generate_response():
     global stop_generation
     stop_generation = False
-
+    model.reset()
     
     data = request.get_json()
     app.logger.info(data)
@@ -175,18 +175,6 @@ def generate_response():
     return_full_text = parameters.get("return_full_text", False)
 
     
-    model = Llama(
-        model_path=model_name,
-        n_ctx=2000,
-        n_parts=1,
-        #n_batch=100,
-        logits_all=True,
-        #n_threads=12,
-        verbose=True,
-        n_gpu_layers=30,
-        n_gqa=8       #must be set for 70b models
-    )
-    
     # Generate the response
     #system_tokens = get_system_tokens(model)
     #tokens = system_tokens