Can't load finetuned model properly.
I am making a model for joint bert. After trained my model, i try to eval before saving and it gives with %95 accuracy. But the problem is when i save this trained model and load it, i get the awful result. I hope you can help me about finding why i cant load properly.
Here is some part of my code

```
class JointBertClassification(BertPreTrainedModel):
    def __init__(self, model_name, config, num_intent_labels, num_slot_labels, args):
        super(JointBertClassification, self).__init__(config)
        self.num_intent_labels = num_intent_labels
        self.num_slot_labels = num_slot_labels
        
        dropout_rate = args[
            "dropout_rate"
        ]
        self.bert = BertModel.from_pretrained(
            model_name, config=self.bert_config
        )  # Load pretrained bert
        self.intent_classifier = IntentClassifier(
            config.hidden_size, num_intent_labels, dropout_rate
        )
        self.slot_classifier = SlotClassifier(
            config.hidden_size, num_slot_labels, dropout_rate
        )
        # self.init_weights()

    def forward(
        self,
        input_ids,
        attention_mask,
        token_type_ids,
        intent_label_ids,
        slot_label_ids,
    ):
        ...
```

```
class JointModel:
    def __init__(
        self, model_type, model_name, intents=None, slots=None, args=None, use_cuda=None
    ):
        """
        Initializes a Joint Model
        
        Args: 
            model_type: The type of model 
            model_name: Default Transformer model name or path to directory containing Transformer model file 
            intents (optional): A list of all Intent labels. If not given ATIS intents will set as default.
            slots (optional): A list of all Slot labels. If not given ATIS slots will set as default.
            args (optional): Default args will be used if thi parameter is not provided. If provided, it should be a dict containing the args that should be changed in the default args.
            use_cuda (optional): Use GPU if available. Setting to False will force model to use CPU only.
        """

        MODEL_CLASSES = {"bert": (BertConfig, JointBertClassification, BertTokenizer)}

        self.config_class, self.model_class, tokenizer_class = MODEL_CLASSES[model_type]

        if intents:
            self.intent_labels = intents
        else:
            self.intent_labels = pd.read_csv(
                "jointbert/data/atis/vocab.intent", header=None, index_col=0
            ).index.tolist()
        self.num_intents = len(self.intent_labels)

        if slots:
            self.slot_labels = slots
        else:
            self.slot_labels = pd.read_csv(
                "jointbert/data/atis/vocab.slot", header=None, index_col=0
            ).index.tolist()
        self.num_slots = len(self.slot_labels)

        self.tokenizer = tokenizer_class.from_pretrained(model_name)
        self.bert_config = self.config_class.from_pretrained(model_name)
       
        self.model = self.model_class(
            model_name,
            self.bert_config,
            num_slot_labels=self.num_slots,
            num_intent_labels=self.num_intents,
            args={"dropout_rate": 0.2},
        )
       
        if use_cuda:
            if torch.cuda.is_available():
                self.device = torch.device("cuda")
            else:
                raise ValueError(
                    "'use_cuda' set to True when cuda is unavaiable. Make sure CUDA is avaiable or set use_cuda=False"
                )
        else:
            self.device = "cpu"

        self.results = {}
        self.args = {
            "output_dir": "outputs/",
            "cache_dir": "cache_dir/",
            "fp16": False,
            "fp16_opt_level": "O1",
            "max_seq_length": 128,
            "train_batch_size": 8,
            "gradient_accumulation_steps": 1,
            "eval_batch_size": 8,
            "num_train_epochs": 1,
            "weight_decay": 0,
            "learning_rate": 4e-5,
            "adam_epsilon": 1e-8,
            "warmup_ratio": 0.06,
            "warmup_steps": 0,
            "max_grad_norm": 1.0,
            "logging_steps": 50,
            "save_steps": 2000,
            "evaluate_during_training": False,
            "overwrite_output_dir": False,
            "reprocess_input_data": False,
            "process_count": 1,
            "n_gpu": 1,
            "silent": False,
        }

        if args:
            self.args.update(args)

        self.args["model_name"] = model_name
        self.args["model_type"] = model_type

        self.pad_token_label_id = CrossEntropyLoss().ignore_index
```

The Saving Part after training
```
    def train_model(
        self,
        train_data,
        output_dir=None,
        show_running_loss=True,
        args=None,
        eval_df=None,
    ):
        if args:
            self.args.update(args)

        if self.args["silent"]:
            show_running_loss = False

        if not output_dir:
            output_dir = self.args["output_dir"]

        if (
            os.path.exists(output_dir)
            and os.listdir(output_dir)
            and not self.args["overwrite_output_dir"]
        ):
            raise ValueError("--")

        self._move_model_to_device()
        train_dataset = self.load_and_cache_examples(train_data)

        global_set, tr_loss = self.train(
            train_dataset,
            output_dir,
            show_running_loss=show_running_loss,
            eval_df=eval_df,
        )

        if not os.path.exists(output_dir):
            os.makedirs(output_dir)

        model_to_save = (
            self.model.module if hasattr(self.model, "module") else self.model
        )
        
        model_to_save.save_pretrained(output_dir)
        self.tokenizer.save_pretrained(output_dir)
        torch.save(self.args, os.path.join(output_dir, "training_args.bin"))
        print(
            "Training of {} model complete. Saved to {}. Training Loss : {}".format(
                self.args["model_type"], output_dir, tr_loss
            )
        )

```
Hi, I met the same problem as you mentioned! Do you fix it? 
my question is here, https://github.com/huggingface/transformers/issues/2402<|||||>No, not yet :/ @trueto 
I think it saves somehow wrong model but i am not sure. <|||||>This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions.
