trainer throw "Torch not compiled with CUDA enabled"
### System Info

transformers version: 4.31.0.dev0
Platform: macOS-13.2.1-arm64-arm-64bit
Python version: 3.10.10
Huggingface_hub version: 0.16.4
Safetensors version: 0.3.1
Accelerate version: 0.22.0.dev0
Accelerate config: not found
PyTorch version (GPU?): 2.0.0 (False)
Tensorflow version (GPU?): 2.10.0 (True)
Flax version (CPU?/GPU?/TPU?): not installed (NA)
Jax version: not installed
JaxLib version: not installed
Using GPU in script?:
Using distributed or parallel set-up in script?:
Accelerate version: 0.22.0.dev0

### Who can help?

@younesbelkada, @sgugger

### Information

- [ ] The official example scripts
- [X] My own modified scripts

### Tasks

- [ ] An officially supported task in the `examples` folder (such as GLUE/SQuAD, ...)
- [ ] My own task or dataset (give details below)

### Reproduction

I'm running following code in Macbook M2 Pro. 
I can run this code with T4 GPU but when I try to run it on my Apple Macbook, it is throwing error, "Torch not compiled with CUDA enabled". I can run other models with my MAC. GPU utilization is good. I suspect that I forgot to change some parameters related to MPS. Thanks in advance.

```
import transformers
from transformers import LlamaTokenizer, LlamaForCausalLM

from typing import List
import polars as pl
import html
import datasets
import torch
from datasets import load_dataset
from datasets import Dataset
import pandas as pd

DEVICE = "mps"

CUTOFF_LEN = 256
safety_param = 10

import pandas as  pd
df = pd.read_parquet("zkp_training_data.parquet")

data_filtered = (
    pl.from_pandas(df).with_columns(pl.when(pl.col("description").is_null()).then("").otherwise(pl.col("description") + ". ").alias("description"))
                .with_columns( (pl.col("description") + pl.col("readme")).alias("final_text") )
                .select(["repo_id",pl.col("final_text"), pl.col("label")])
).to_pandas()

data_filtered.loc[data_filtered.label == "other",["label"]] = "not zero-knowledge-proof"

dataset = datasets.Dataset.from_pandas(data_filtered)
dataset_rev = dataset.shuffle(seed=42)
dataset_final = dataset_rev.class_encode_column("label").train_test_split(stratify_by_column="label",train_size=0.8)

def map_func(example):
    mapping=dataset_final["train"].features["label"]
    val_set = mapping.int2str(example["label"])
    del example["label"]
    example["labels"] = val_set
    return example

dataset_final_v2 = dataset_final.map(map_func,batched=True)

BASE_MODEL = "decapoda-research/llama-7b-hf"
 
model = LlamaForCausalLM.from_pretrained(
    BASE_MODEL,
    load_in_8bit=True,
    torch_dtype=torch.float16,
    device_map="auto",
)

tokenizer = LlamaTokenizer.from_pretrained("decapoda-research/llama-7b-hf")
 
tokenizer.pad_token_id = (
    0  # unk. we want this to be different from the eos token
)
tokenizer.padding_side = "left"


def generate_dummy_prompt_v2(input,output):
    partial_string = f"""Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.  # noqa: E501
### Instruction:
Does this project include zero-knowledge-proof implementation?
### Response:
{output}"""

    full_string = html.unescape(partial_string)

    result = tokenizer(
        full_string,
        padding=False)
    
    constant_part_token_len = len(result["input_ids"])
    
    input_html = html.unescape(input)
    
    result_input = tokenizer(
        input,
        padding=False)
    
    input_token_len = len(result_input["input_ids"])
    allowed_len = (CUTOFF_LEN - constant_part_token_len) - safety_param
    input = tokenizer.decode(result_input["input_ids"][:allowed_len],skip_special_tokens = True)

    final_prompt = f"""Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.  # noqa: E501
### Instruction:
Does this project include zero-knowledge-proof implementation?
### Input:
{input}
### Response:
{output}
"""
    
    final_prompt = html.unescape(final_prompt)
    if len(final_prompt.split(" "))<3:
        print("aaa")
    return final_prompt


dataset_final_v3 = dataset_final_v2.map(
    lambda x: {"final_prompt": [generate_dummy_prompt_v2(a,b) for a,b in zip(x["final_text"],x["labels"]) ]}, batched=True
)


def tokenize(prompt, add_eos_token=True):
    result = tokenizer(
        prompt["final_prompt"],
        truncation=True,
        max_length=CUTOFF_LEN,
        padding=False,
        return_tensors=None,
    )
    if (
        result["input_ids"][-1] != tokenizer.eos_token_id
        and len(result["input_ids"]) < CUTOFF_LEN
        and add_eos_token
    ):
        result["input_ids"].append(tokenizer.eos_token_id)
        result["attention_mask"].append(1)
 
    result["labels"] = result["input_ids"].copy()
 
    return result

dataset_final_v4 = dataset_final_v3.map(tokenize,batched=False,remove_columns=["repo_id","final_text","final_prompt"])
dataset_final_v4.save_to_disk("dataset_final_v4")

train_data = dataset_final_v4["train"]
val_data = dataset_final_v4["test"]

LORA_R = 8
LORA_ALPHA = 16
LORA_DROPOUT= 0.05
LORA_TARGET_MODULES = [
    "q_proj",
    "v_proj",
]
 
BATCH_SIZE = 128
MICRO_BATCH_SIZE = 4
GRADIENT_ACCUMULATION_STEPS = BATCH_SIZE // MICRO_BATCH_SIZE
LEARNING_RATE = 3e-4
TRAIN_STEPS = 300
OUTPUT_DIR = "experiments_rev"


training_arguments = transformers.TrainingArguments(
    per_device_train_batch_size=MICRO_BATCH_SIZE,
    gradient_accumulation_steps=GRADIENT_ACCUMULATION_STEPS,
    warmup_steps=100,
    max_steps=TRAIN_STEPS,
    learning_rate=LEARNING_RATE,
    fp16=False,
    logging_steps=2,
    optim="adamw_torch",
    evaluation_strategy="steps",
    save_strategy="steps",
    eval_steps=4,
    save_steps=24,
    output_dir=OUTPUT_DIR,
    use_mps_device = True,
    save_total_limit=3,
    overwrite_output_dir=True,
    report_to="tensorboard"
)

data_collator = transformers.DataCollatorForSeq2Seq(
    tokenizer, pad_to_multiple_of=8, return_tensors="pt", padding=True
)

trainer = transformers.Trainer(
    model=model,
    train_dataset=train_data,
    eval_dataset=val_data,
    args=training_arguments,
    data_collator=data_collator
)
model.config.use_cache = False

model = torch.compile(model)
trainer.train(resume_from_checkpoint = False)
model.save_pretrained(OUTPUT_DIR)

```

### Expected behavior

it should start to train data but it throws error when trainer.train is executed. I suspect mps is not supporting some features of model.
Hi @nemesis00sam 
Thanks for the issue, there are two issues with your training setup
1- you are trying to load your model in 8bit, which is only supported on GPU devices (not M1 chips)
2- It seems you are trying to do pure 8bit training, despite I see lora-specific arguments, i don't see where they are used. If you want to train 8bit models, consider converting your model into a PeftModel and train adapters on it for example: https://github.com/huggingface/peft/blob/main/examples/int8_training/Finetune_opt_bnb_peft.ipynb<|||||>Thanks @younesbelkada. I'm a little bit confused. I need more explanation. Sorry for taking your time. load_in_8bit=True will not work for m1? Should I convert it to 8bit with different package?