Update app.py

app.py

@@ -21,3 +21,49 @@ base_model_name = "meta-llama/Llama-3.1-8B"
 # Your fine-tuned LoRA adapter (uploaded to Hugging Face)
 lora_model_name = "starnernj/Early-Christian-Church-Fathers-LLaMA-3.1-Fine-Tuned"
 
+# Login because LLaMA 3.1 8B is a gated model
+login(token=os.getenv("HuggingFaceFineGrainedReadToken"))
+print("After login")
+# Enable 4-bit Quantization with BitsAndBytes
+quantization_config = BitsAndBytesConfig(
+    load_in_4bit=True,  # ✅ Enables 4-bit quantization for memory efficiency
+    bnb_4bit_compute_dtype=torch.float16,  # ✅ Uses float16 for performance
+    bnb_4bit_use_double_quant=True,  # ✅ Optimizes quantization
+    bnb_4bit_quant_type="nf4"  # ✅ Normalized Float-4 for better accuracy
+)
+
+base_model = AutoModelForCausalLM.from_pretrained(
+    base_model_name,
+    quantization_config=quantization_config,
+    device_map="auto"
+)
+print("After basemodel")
+# Load tokenizer
+tokenizer = AutoTokenizer.from_pretrained(base_model_name)
+print("After Tokenizer")
+# Load LoRA Adapter
+print("Loading LoRA adapter...")
+model = PeftModel.from_pretrained(base_model, lora_model_name)
+print("After peft model")
+# Function to generate responses
+def chatbot_response(user_input):
+    try:
+        inputs = tokenizer(user_input, return_tensors="pt").to("cuda")
+        outputs = model.generate(**inputs, max_length=200)
+        return tokenizer.decode(outputs[0], skip_special_tokens=True)
+    except Exception as e:
+        error_message = f"AssertionError: {str(e)}\n{traceback.format_exc()}"
+        print(error_message)  # ✅ Logs detailed error messages
+        return "An error occurred. Check the logs for details."
+
+
+# Launch the Gradio chatbot
+interface = gr.Interface(
+    fn=chatbot_response,
+    inputs=gr.Textbox(lines=2, placeholder="Ask me about the Christian Church Fathers..."),
+    outputs="text",
+    title="Early Christian Church Fathers Fine-Tuned LLaMA 3.1 8B with LoRA",
+    description="A chatbot using a fine-tuned LoRA adapter on LLaMA 3.1 8B, tuned on thousands of writings of the early Christian Church Fathers.",
+)
+
+interface.launch()