npvinHnivqn/Mistral-7B-Instruct

Quick start

pip install accelerate peft bitsandbytes pip install git+https://github.com/huggingface/transformers trl py7zr auto-gptq optimum

from peft import AutoPeftModelForCausalLM
from transformers import GenerationConfig
from transformers import AutoTokenizer
import torch
tokenizer = AutoTokenizer.from_pretrained("npvinHnivqn/Mistral-7B-Instruct")


model = AutoPeftModelForCausalLM.from_pretrained(
    'npvinHnivqn/Mistral-7B-Instruct',
    low_cpu_mem_usage=True,
    return_dict=True,
    torch_dtype=torch.float16,
    device_map="cuda")

generation_config = GenerationConfig(
    do_sample=True,
    top_k=1,
    temperature=0.1,
    max_new_tokens=25,
    pad_token_id=tokenizer.eos_token_id
)

inputs = tokenizer("""<|SYSTEM|> You are a very good chatbot, you can answer every question from users. <|USER|> Summarize this following dialogue: Vasanth: I'm at the railway station in Chennai Karthik: No problems so far? Vasanth: no, everything's going smoothly Karthik: good. lets meet there soon! [INPUT]  <|BOT|>""", return_tensors="pt").to("cuda")
outputs = model.generate(**inputs, generation_config=generation_config)
print(tokenizer.decode(outputs[0], skip_special_tokens=True))