ModelUtilsTest.test_model_from_pretrained failiing on CUDA
Seems as though an `architecture` key is being added.

Not sure who to assign this to @LysandreJik.
```python
__________________ ModelUtilsTest.test_model_from_pretrained ___________________
[gw0] linux -- Python 3.7.6 /home/hf/actions-runner_transformers/_work/transformers/transformers/.env/bin/python

self = <tests.test_modeling_common.ModelUtilsTest testMethod=test_model_from_pretrained>

    @slow
    def test_model_from_pretrained(self):
        for model_name in BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            config = BertConfig.from_pretrained(model_name)
            self.assertIsNotNone(config)
            self.assertIsInstance(config, PretrainedConfig)
    
            model = BertModel.from_pretrained(model_name)
            model, loading_info = BertModel.from_pretrained(model_name, output_loading_info=True)
            self.assertIsNotNone(model)
            self.assertIsInstance(model, PreTrainedModel)
            for value in loading_info.values():
                self.assertEqual(len(value), 0)
    
            config = BertConfig.from_pretrained(model_name, output_attentions=True, output_hidden_states=True)
            model = BertModel.from_pretrained(model_name, output_attentions=True, output_hidden_states=True)
            self.assertEqual(model.config.output_hidden_states, True)
>           self.assertEqual(model.config, config)
E           AssertionError: BertConfig {
E             "_name_or_path": "bert-base-uncased",
E             "a[518 chars]22
E           }
E            != BertConfig {
E             "architectures": [
E               "BertForMaskedLM"
E            [478 chars]22
E           }

tests/test_modeling_common.py:1092: AssertionError

```

https://github.com/huggingface/transformers/runs/1303479917?check_suite_focus=true
Assign me!<|||||>This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions.
