Test w/o 8bit

app.py

@@ -7,7 +7,7 @@ checkpoint = 1200
 
 # Load your fine-tuned model and tokenizer
 tokenizer = t.AutoTokenizer.from_pretrained("NousResearch/Llama-2-7b-hf")
-model = t.AutoModelForCausalLM.from_pretrained("NousResearch/Llama-2-7b-hf", load_in_8bit=True, torch_dtype=torch.float16)
+model = t.AutoModelForCausalLM.from_pretrained("NousResearch/Llama-2-7b-hf")
 tokenizer.pad_token_id = 0
 
 config = peft.LoraConfig(r=8, lora_alpha=16, target_modules=["q_proj", "v_proj"], lora_dropout=0.005, bias="none", task_type="CAUSAL_LM")