cpu

config.py

@@ -4,7 +4,7 @@ from typing import List, Optional
 # Hyperparameters for Model
 max_seq_length = 2048 # Choose any! We auto support RoPE Scaling internally!
 dtype = None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+
-load_in_4bit = True # Use 4bit quantization to reduce memory usage. Can be False.
+load_in_4bit = False # Use 4bit quantization to reduce memory usage. Can be False.
 lora_r = 16 # Number of attention heads for LoRA
 lora_alpha = 16 # Alpha value for LoRA
 lora_dropout = 0 # Dropout rate for LoRA

inference.py

@@ -24,7 +24,7 @@ def load_model(
 
     kwargs = {
         "device_map": "cpu",
-        "torch_dtype": torch.float32,
+        "torch_dtype": dtype,
         "low_cpu_mem_usage": True,
         "_from_auto": False,  # Prevent automatic quantization detection
         "quantization_config": None  # Explicitly set no quantization
@@ -34,7 +34,7 @@ def load_model(
 
     model = AutoModelForCausalLM.from_pretrained(
         pretrained_model_name_or_path=model_name,
-        config = kwargs
+        **kwargs
     )
 
     model.eval() # Set model to evaluation mode
@@ -57,12 +57,14 @@ def prepare_input(
     Returns:
         torch.Tensor: Prepared input tensor
     """
-    return tokenizer.apply_chat_template(
+    return tokenizer(
         messages,
-        tokenize=True,
-        add_generation_prompt=True,
-        return_tensors="pt"
-    ).to(device)
+        # tokenize=True,
+        # add_generation_prompt=True,
+        return_tensors="pt",
+        padding=True,
+        truncation=True,
+    )["input_ids"]
 
 def generate_response(
     model: AutoModelForCausalLM,