can't pickle <class 'types.AutoModelForCausalLM'>

Hi, a new problem has arisen
we can pickle "LazyModule" now, but can't  pickle <class 'types.AutoModelForCausalLM'>

@stas00 @patrickvonplaten, @LysandreJik




Traceback (most recent call last):
  File "/usr/local/anaconda3/envs/py38/lib/python3.8/multiprocessing/process.py", line 315, in _bootstrap
    self.run()
  File "/usr/local/anaconda3/envs/py38/lib/python3.8/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "/media/cfs/gonglixing/9Nctl/gpt_v2/run_clm_v3.py", line 509, in init_process
    fn(rank, size)
  File "/media/cfs/gonglixing/9Nctl/gpt_v2/run_clm_v3.py", line 367, in main
    tokenized_datasets = raw_datasets.map(
  File "/usr/local/anaconda3/envs/py38/lib/python3.8/site-packages/datasets/dataset_dict.py", line 471, in map
    {
  File "/usr/local/anaconda3/envs/py38/lib/python3.8/site-packages/datasets/dataset_dict.py", line 472, in <dictcomp>
    k: dataset.map(
  File "/usr/local/anaconda3/envs/py38/lib/python3.8/site-packages/datasets/arrow_dataset.py", line 1736, in map
    transformed_shards = [r.get() for r in results]
  File "/usr/local/anaconda3/envs/py38/lib/python3.8/site-packages/datasets/arrow_dataset.py", line 1736, in <listcomp>
    transformed_shards = [r.get() for r in results]
  File "/usr/local/anaconda3/envs/py38/lib/python3.8/site-packages/multiprocess/pool.py", line 771, in get
    raise self._value
  File "/usr/local/anaconda3/envs/py38/lib/python3.8/site-packages/multiprocess/pool.py", line 537, in _handle_tasks
    put(task)
  File "/usr/local/anaconda3/envs/py38/lib/python3.8/site-packages/multiprocess/connection.py", line 209, in send
    self._send_bytes(_ForkingPickler.dumps(obj))
  File "/usr/local/anaconda3/envs/py38/lib/python3.8/site-packages/multiprocess/reduction.py", line 54, in dumps
    cls(buf, protocol, *args, **kwds).dump(obj)
  File "/usr/local/anaconda3/envs/py38/lib/python3.8/site-packages/dill/_dill.py",line 498, in dump
    StockPickler.dump(self, obj)
  File "/usr/local/anaconda3/envs/py38/lib/python3.8/pickle.py", line 487, in dump
    self.save(obj)
  File "/usr/local/anaconda3/envs/py38/lib/python3.8/pickle.py", line 560, in save
    f(self, obj)  # Call unbound method with explicit self
  File "/usr/local/anaconda3/envs/py38/lib/python3.8/pickle.py", line 901, in save_tuple
    save(element)
  File "/usr/local/anaconda3/envs/py38/lib/python3.8/pickle.py", line 560, in save
    f(self, obj)  # Call unbound method with explicit self
  File "/usr/local/anaconda3/envs/py38/lib/python3.8/site-packages/dill/_dill.py",line 990, in save_module_dict
    StockPickler.save_dict(pickler, obj)
  File "/usr/local/anaconda3/envs/py38/lib/python3.8/pickle.py", line 971, in save_dict
    self._batch_setitems(obj.items())
  File "/usr/local/anaconda3/envs/py38/lib/python3.8/pickle.py", line 997, in _batch_setitems
    save(v)
  File "/usr/local/anaconda3/envs/py38/lib/python3.8/pickle.py", line 560, in save
    f(self, obj)  # Call unbound method with explicit self
  File "/usr/local/anaconda3/envs/py38/lib/python3.8/site-packages/dill/_dill.py",line 1493, in save_function
    pickler.save_reduce(_create_function, (obj.__code__,
  File "/usr/local/anaconda3/envs/py38/lib/python3.8/pickle.py", line 692, in save_reduce
    save(args)
  File "/usr/local/anaconda3/envs/py38/lib/python3.8/pickle.py", line 560, in save
    f(self, obj)  # Call unbound method with explicit self
  File "/usr/local/anaconda3/envs/py38/lib/python3.8/pickle.py", line 901, in save_tuple
    save(element)
  File "/usr/local/anaconda3/envs/py38/lib/python3.8/pickle.py", line 560, in save
    f(self, obj)  # Call unbound method with explicit self
  File "/usr/local/anaconda3/envs/py38/lib/python3.8/site-packages/dill/_dill.py",line 990, in save_module_dict
    StockPickler.save_dict(pickler, obj)
  File "/usr/local/anaconda3/envs/py38/lib/python3.8/pickle.py", line 971, in save_dict
    self._batch_setitems(obj.items())
  File "/usr/local/anaconda3/envs/py38/lib/python3.8/pickle.py", line 997, in _batch_setitems
    save(v)
  File "/usr/local/anaconda3/envs/py38/lib/python3.8/pickle.py", line 560, in save
    f(self, obj)  # Call unbound method with explicit self
  File "/usr/local/anaconda3/envs/py38/lib/python3.8/site-packages/dill/_dill.py",line 1439, in save_type
    StockPickler.save_global(pickler, obj, name=name)
  File "/usr/local/anaconda3/envs/py38/lib/python3.8/pickle.py", line 1070, in save_global
    raise PicklingError(
_pickle.PicklingError: Can't pickle <class 'types.AutoModelForCausalLM'>: it's notfound as types.AutoModelForCausalLM

_Originally posted by @lancekung in https://github.com/huggingface/transformers/issues/12549#issuecomment-877537851_
Hello! Could you provide a code example that yields this error? Thank you!<|||||>```
import pickle
from transformers import AutoModelForCausalLM

pickle.dumps(AutoModelForCausalLM)
```
I think it's comes from the fact those are autogenerated.<|||||>> ```
> import pickle
> from transformers import AutoModelForCausalLM
> 
> pickle.dumps(AutoModelForCausalLM)
> ```
> 
> I think it's comes from the fact those are autogenerated.

thanks for your help, but I tested based on your modification in #12654, a new problem arises：




Traceback (most recent call last):
  File "/usr/local/anaconda3/envs/py38/lib/python3.8/multiprocessing/process.py", line 315, in _bootstrap
    self.run()
  File "/usr/local/anaconda3/envs/py38/lib/python3.8/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "/media/cfs/gonglixing/9Nctl/gpt_v2/run_clm_v3.py", line 509, in init_process
    fn(rank, size)
  File "/media/cfs/gonglixing/9Nctl/gpt_v2/run_clm_v3.py", line 456, in main
    train_result = trainer.train(resume_from_checkpoint=checkpoint)
  File "/media/cfs/gonglixing/9Nctl/opensource/transformers-master/src/transformers/trainer.py", line 1275, in train
    tr_loss += self.training_step(model, inputs)
  File "/media/cfs/gonglixing/9Nctl/opensource/transformers-master/src/transformers/trainer.py", line 1778, in training_step
    self.scaler.scale(loss).backward()
  File "/usr/local/anaconda3/envs/py38/lib/python3.8/site-packages/torch/tensor.py", line 245, in backward
    torch.autograd.backward(self, gradient, retain_graph, create_graph, inputs=inputs)
  File "/usr/local/anaconda3/envs/py38/lib/python3.8/site-packages/torch/autograd/__init__.py", line 145, in backward
    Variable._execution_engine.run_backward(
SystemError: <built-in method run_backward of torch._C._EngineBase object at 0x7f06bfae6b30> returned NULL without setting an error

