4bit Blip2 compatibility
### System Info

I am getting an error after loading Blip2 in 4bit, cant inference, cant train. Can anyone help?


### Who can help?

_No response_

### Information

- [ ] The official example scripts
- [ ] My own modified scripts

### Tasks

- [ ] An officially supported task in the `examples` folder (such as GLUE/SQuAD, ...)
- [ ] My own task or dataset (give details below)

### Reproduction

`import torch
from transformers import Blip2ForConditionalGeneration, AutoProcessor, Blip2Processor, AutoModelForCausalLM, BitsAndBytesConfig
from peft import prepare_model_for_kbit_training

#processor = AutoProcessor.from_pretrained("Salesforce/blip2-opt-6.7b-coco")
#model = Blip2ForConditionalGeneration.from_pretrained("Salesforce/blip2-opt-6.7b-coco", device_map='auto', load_in_8bit=True)

nf4_config = BitsAndBytesConfig(
   load_in_4bit=True,
   bnb_4bit_quant_type="nf4",
   bnb_4bit_use_double_quant=True,
   bnb_4bit_compute_dtype=torch.bfloat16
)

processor = AutoProcessor.from_pretrained("Salesforce/blip2-opt-6.7b-coco")
model = Blip2ForConditionalGeneration.from_pretrained("Salesforce/blip2-opt-6.7b-coco", device_map='auto', quantization_config=nf4_config)`

Then when I want to train with PEFT or just do a single image captioning with the loaded model I get:

`FP4 quantization state not initialized. Please call .cuda() or .to(device) on the LinearFP4 layer first.
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
Cell In[10], line 6
      3 pixel_values = inputs.pixel_values
      5 #generated_ids = model.generate(pixel_values=pixel_values, min_length=50, max_new_tokens=50, length_penalty=1.4, top_k=150, top_p=0.95, repetition_penalty=2.1, num_beams=5, temperature=0.75)
----> 6 generated_ids = model.generate(pixel_values=pixel_values, max_length=50)
      7 generated_caption = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
      8 print(generated_caption)

File H:\CONDA\envs\blip\lib\site-packages\torch\utils\_contextlib.py:115, in context_decorator.<locals>.decorate_context(*args, **kwargs)
    112 @functools.wraps(func)
    113 def decorate_context(*args, **kwargs):
    114     with ctx_factory():
--> 115         return func(*args, **kwargs)

File H:\CONDA\envs\blip\lib\site-packages\transformers\models\blip_2\modeling_blip_2.py:1854, in Blip2ForConditionalGeneration.generate(self, pixel_values, input_ids, attention_mask, **generate_kwargs)
   1851 inputs_embeds = self.get_input_embeddings()(input_ids)
   1852 inputs_embeds = torch.cat([language_model_inputs, inputs_embeds.to(language_model_inputs.device)], dim=1)
-> 1854 outputs = self.language_model.generate(
   1855     inputs_embeds=inputs_embeds,
   1856     attention_mask=attention_mask,
   1857     **generate_kwargs,
   1858 )
   1860 return outputs

File H:\CONDA\envs\blip\lib\site-packages\torch\utils\_contextlib.py:115, in context_decorator.<locals>.decorate_context(*args, **kwargs)
    112 @functools.wraps(func)
    113 def decorate_context(*args, **kwargs):
    114     with ctx_factory():
--> 115         return func(*args, **kwargs)

File H:\CONDA\envs\blip\lib\site-packages\transformers\generation\utils.py:1518, in GenerationMixin.generate(self, inputs, generation_config, logits_processor, stopping_criteria, prefix_allowed_tokens_fn, synced_gpus, assistant_model, streamer, **kwargs)
   1512         raise ValueError(
   1513             "num_return_sequences has to be 1 when doing greedy search, "
   1514             f"but is {generation_config.num_return_sequences}."
   1515         )
   1517     # 11. run greedy search
-> 1518     return self.greedy_search(
   1519         input_ids,
   1520         logits_processor=logits_processor,
   1521         stopping_criteria=stopping_criteria,
   1522         pad_token_id=generation_config.pad_token_id,
   1523         eos_token_id=generation_config.eos_token_id,
   1524         output_scores=generation_config.output_scores,
   1525         return_dict_in_generate=generation_config.return_dict_in_generate,
   1526         synced_gpus=synced_gpus,
   1527         streamer=streamer,
   1528         **model_kwargs,
   1529     )
   1531 elif is_contrastive_search_gen_mode:
   1532     if generation_config.num_return_sequences > 1:

File H:\CONDA\envs\blip\lib\site-packages\transformers\generation\utils.py:2335, in GenerationMixin.greedy_search(self, input_ids, logits_processor, stopping_criteria, max_length, pad_token_id, eos_token_id, output_attentions, output_hidden_states, output_scores, return_dict_in_generate, synced_gpus, streamer, **model_kwargs)
   2332 model_inputs = self.prepare_inputs_for_generation(input_ids, **model_kwargs)
   2334 # forward pass to get next token
-> 2335 outputs = self(
   2336     **model_inputs,
   2337     return_dict=True,
   2338     output_attentions=output_attentions,
   2339     output_hidden_states=output_hidden_states,
   2340 )
   2342 if synced_gpus and this_peer_finished:
   2343     continue  # don't waste resources running the code we don't need

File H:\CONDA\envs\blip\lib\site-packages\torch\nn\modules\module.py:1501, in Module._call_impl(self, *args, **kwargs)
   1496 # If we don't have any hooks, we want to skip the rest of the logic in
   1497 # this function, and just call forward.
   1498 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1499         or _global_backward_pre_hooks or _global_backward_hooks
   1500         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1501     return forward_call(*args, **kwargs)
   1502 # Do not call functions when jit is used
   1503 full_backward_hooks, non_full_backward_hooks = [], []

File H:\CONDA\envs\blip\lib\site-packages\accelerate\hooks.py:165, in add_hook_to_module.<locals>.new_forward(*args, **kwargs)
    163         output = old_forward(*args, **kwargs)
    164 else:
--> 165     output = old_forward(*args, **kwargs)
    166 return module._hf_hook.post_forward(module, output)

File H:\CONDA\envs\blip\lib\site-packages\transformers\models\opt\modeling_opt.py:957, in OPTForCausalLM.forward(self, input_ids, attention_mask, head_mask, past_key_values, inputs_embeds, labels, use_cache, output_attentions, output_hidden_states, return_dict)
    944 # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
    945 outputs = self.model.decoder(
    946     input_ids=input_ids,
    947     attention_mask=attention_mask,
   (...)
    954     return_dict=return_dict,
    955 )
--> 957 logits = self.lm_head(outputs[0]).contiguous()
    959 loss = None
    960 if labels is not None:
    961     # move labels to correct device to enable model parallelism

File H:\CONDA\envs\blip\lib\site-packages\torch\nn\modules\module.py:1501, in Module._call_impl(self, *args, **kwargs)
   1496 # If we don't have any hooks, we want to skip the rest of the logic in
   1497 # this function, and just call forward.
   1498 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1499         or _global_backward_pre_hooks or _global_backward_hooks
   1500         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1501     return forward_call(*args, **kwargs)
   1502 # Do not call functions when jit is used
   1503 full_backward_hooks, non_full_backward_hooks = [], []

File H:\CONDA\envs\blip\lib\site-packages\accelerate\hooks.py:165, in add_hook_to_module.<locals>.new_forward(*args, **kwargs)
    163         output = old_forward(*args, **kwargs)
    164 else:
--> 165     output = old_forward(*args, **kwargs)
    166 return module._hf_hook.post_forward(module, output)

File H:\CONDA\envs\blip\lib\site-packages\bitsandbytes\nn\modules.py:219, in Linear4bit.forward(self, x)
    216     x = x.to(self.compute_dtype)
    218 bias = None if self.bias is None else self.bias.to(self.compute_dtype)
--> 219 out = bnb.matmul_4bit(x, self.weight.t(), bias=bias, quant_state=self.weight.quant_state)
    221 out = out.to(inp_dtype)
    223 return out

AttributeError: 'Parameter' object has no attribute 'quant_state'`



### Expected behavior

8 bit works fine
cc @younesbelkada <|||||>Hi @betterftr 
Thanks for the issue, indeed there seems to be a bug, that should be fixed in https://github.com/huggingface/transformers/pull/23895