Hi everyone,
I have a doubt that haunts me and I cannot clarify myself. I have two model for fine tuning a BERT model.
The first model follow exactly the run_ner script and use from transformers import AutoModelForTokenClassification where the model is "Musixmatch/umberto-wikipedia-uncased-v1" (BERT for Italian, but I think that is not important which type of BERT I used).
The second model is the follow:
class Bert(object):
    """A facade to Bert model that extracts features for sets of tokens"""

    def __init__(self, layer_indexes, max_seq_length, batch_size, multi_lingual=False, which_cuda = 0):
        pretrained_model =  "Musixmatch/umberto-wikipedia-uncased-v1"

       ... setting all parameters self. ...

        self.tokenizer = AutoTokenizer.from_pretrained(pretrained_model, do_lower_case=not multi_lingual)
        self.model = AutoModel.from_pretrained(pretrained_model).to(self.device)
        # tells pytorch to run in evaluation mode instead of training
        self.model.eval() 
        
    def get_bert_features(self, sentence):
        ## sentence is in the format ['tok1', 'tok2']
        bert_tokens, map_to_original_tokens = self.convert_to_bert_tokenization(sentence)
        feature = self.from_bert_tokens_to_features(bert_tokens, map_to_original_tokens)
        features = [feature]
        
        # get ids
        all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
        # mask with 0's for placeholders
        all_input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long)
        # tensor with 1...n where n is the number of examples
        all_encoder_layers, _ = self.model(all_input_ids, token_type_ids=None, attention_mask=all_input_mask)
        last_layer = all_encoder_layers[-1]
        
        return bert_tokens, map_to_original_tokens, last_layer

    def extract_bert_features(self, conll_dataset):
        sentences = [[e.form for e in sentence] for sentence in conll_dataset]
        # data loading
        features = []
        for sentence in sentences:
            bert_tokens, map_to_original_tokens = self.convert_to_bert_tokenization(sentence)
            feature = self.from_bert_tokens_to_features(bert_tokens, map_to_original_tokens)
            features.append(feature)
        
        all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
        # mask with 0's for placeholders
        all_input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long)
        # tensor with 1...n where n is the number of examples
        all_token_maps = torch.tensor([f.map_to_original_tokens for f in features], dtype=torch.long)
        # indexes that map back dataset
        all_example_index = torch.arange(all_input_ids.size(0), dtype=torch.long)
        
        # create a dataset the resources needed
        eval_data = TensorDataset(all_input_ids, all_input_mask, all_token_maps, all_example_index)
        # create a sampler which will be used to create the batches
        eval_sampler = SequentialSampler(eval_data)
        eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=self.batch_size)

        for input_ids, input_mask, token_maps, example_indices in eval_dataloader:
            input_ids = input_ids.to(self.device)
            input_mask = input_mask.to(self.device)
            all_encoder_layers, _ = self.model(input_ids, token_type_ids=None, attention_mask=input_mask)
            averaged_output = all_encoder_layers
           
            for i, idx in enumerate(example_indices):
                for j, coll_entry in enumerate(conll_dataset[idx]):
                    if token_maps[i,j] < 511:
                        coll_entry.bert = averaged_output[i,token_maps[i,j]].clone().detach().cpu()
                    else:
                        coll_entry.bert = averaged_output[i,token_maps[i,511]].clone().detach().cpu()

where I used the same BERT model, but WITHOUT the final fine tuning layer:
(dropout): Dropout(p=0.1, inplace=False)
(classifier): Linear(in_features=768, out_features=X, bias=True)

because I define my fine tuning layer inside a ModelPat in the following way:
class ModelPaT(nn.Module):

    def __init__(self, args, word_vocab, tag_vocab, pos_vocab, deprel_vocab, char_vocab):
        super().__init__()

        ... setting all parameters self. ...

        if self.bert:
            self.bilstm_input_size = self.bert_hidden_size  # 768

	    # FINE TUNING I think...

        self.hidden2_to_pos = nn.Linear(
            in_features=self.bilstm_input_size, # 768 = BERT layer
            out_features=len(self.pos_vocab),
        )

        self.hidden2_to_dep = nn.Linear(
            in_features=self.bilstm_input_size, # 768 = BERT layer 
            out_features=len(self.deprel_vocab),
        )

    def forward(self, sentences):
        orig_w = [[e.form for e in sentence] for sentence in sentences]  # all token from a given sentence
        # print("token: " + str(orig_w))
        w, t, x_lengths = self.sentence2tok_tags(sentences)

        batch_size, seq_len = w.size()
        # (batch_size, seq_len) -> (batch_size, seq_len, embedding_dim)

        if self.bert:
            # get bert features from model
            bert_features_list = [[e.bert for e in sentence] for sentence in sentences]
            # convert list to one tensor
            bert_features_tensor = from_tensor_list_to_one_tensor(bert_features_list, self.bert_hidden_size).to(self.device)

        # (batch_size, seq_len, embedding_dim) -> (batch_size, seq_len, n_lstm_units)
        x = torch.nn.utils.rnn.pack_padded_sequence(bert_features_tensor, x_lengths, batch_first=True)
        x, _ = torch.nn.utils.rnn.pad_packed_sequence(x, batch_first=True)
        # (batch_size, seq_len, n_lstm_units) -> (batch_size * seq_len, n_lstm_units)
        x = x.contiguous()
        x = x.view(-1, x.shape[2])

        y1 = self.hidden2_to_pos(x)
        y2 = self.hidden2_to_dep(x)

        if self.mode == 'evaluation':
            y1 = F.softmax(y1, dim=1)
            y2 = F.softmax(y2, dim=1)

        # (batch_size * seq_len, n_lstm_units) -> (batch_size, seq_len, n_tags)
        y1 = y1.view(batch_size, seq_len, len(self.pos_vocab))
        y2 = y2.view(batch_size, seq_len, len(self.deprel_vocab))

        return y1, y2

where I get:
ModelPat(
  (dropout): Dropout(p=0.1, inplace=False)
  (hidden2_to_pos): Linear(in_features=768, out_features=X, bias=True)
  (hidden2_to_dep): Linear(in_features=768, out_features=Y, bias=True)
)

Now these two models seem to have the same architecture for Bert’s fine tuning (the same model of BERT where at the end I added a Linear layer for fine tuning), but I get very different results with the same dataset. Doing several tests, I doubted that the problem is not so much in the architecture which, in fact, seems to be the same, but in the way in which Bert’s pre-trained models are loaded. Could this be the reason that makes me get different accuracy results?
If so, how could I modify the second model to be similar to the first to achieve the same result?
Thanks a lot to everyone!
P.s. I know that in the second model I have two Linear layers, but they are used to predict different labels and are evaluated separately, so I doubt that is the problem