I’m using the following code
**Import the Libraries : **
! pip install transformers

import pandas as pd
import torch
import torch.nn as nn
from sklearn.metrics import accuracy_score, f1_score
from transformers import AutoModel, BertTokenizer, AdamW
from sklearn.utils.class_weight import compute_class_weight
from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
import numpy as pp

**Prepare the Data : **
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

train_set = pd.read_csv('/content/drive/My Drive/train_set.csv')
val_set = pd.read_csv('/content/drive/My Drive/val_set.csv')
X_train, y_train = train_set['text'], train_set['label']
X_val, y_val = val_set['text'], val_set['label']

# compute the class weights
class_wts = compute_class_weight('balanced', pp.unique(y_train), y_train)

# convert class weights to tensor
weights = torch.tensor(class_wts, dtype=torch.float)
weights = weights.to(device)

tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')

X_train = tokenizer.batch_encode_plus(
    X_train.tolist(),
    truncation=True, return_token_type_ids=False, return_length=True, pad_to_max_length=30,
    return_attention_mask=True)


train_label = torch.tensor(train_set['label'].tolist(), dtype=torch.long)

train_length = torch.tensor(X_train['length']) # I will not use this In the training for now 
train_input = torch.tensor(X_train['input_ids'])
train_attention_mask = torch.tensor(X_train['attention_mask'])
train_set = TensorDataset(train_input, train_length, train_attention_mask, train_label)


X_val = tokenizer.batch_encode_plus(
    X_val.tolist(),
    truncation=True, return_token_type_ids=False, return_length=True, pad_to_max_length=30, return_attention_mask=True)

val_label = torch.tensor(val_set['label'].tolist(), dtype=torch.long)
val_length = torch.tensor(X_val['length']) # I will not use this In the training for now 
val_input = torch.tensor(X_val['input_ids'])
val_attention_mask = torch.tensor(X_val['attention_mask'])
val_set = TensorDataset(val_input, val_length, val_attention_mask, val_label)

train_loader = DataLoader(train_set, batch_size=50, sampler=RandomSampler(train_set))
val_loader = DataLoader(val_set, batch_size=50, sampler=SequentialSampler(val_set))

**Training the model : **
class BERT(nn.Module):
    def __init__(self):
        super(BERT, self).__init__()
        self.bert = AutoModel.from_pretrained('bert-base-uncased')
        for param in self.bert.parameters():
            param.requires_grad = False
        self.linear = nn.Linear(self.bert.config.hidden_size, 200)
        self.drop = nn.Dropout(.5)
        self.output = nn.Linear(200, 2)

    def forward(self, x, mask, len):
        _, x = self.bert(x, attention_mask=mask)
        x = self.drop(x)
        x = nn.functional.relu(self.linear(x))
        x = self.output(x)
        return x

model = BERT()

model.to(device)
weights= torch.tensor(class_wts,dtype=torch.float)
weights = weights.to(device)

optimizer = AdamW(model.parameters(), lr=.01)
cross_entropy = nn.NLLLoss(weight=weights).to(device)
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=.1, patience=2, verbose=True)


def big_training_loop(epochs, optim, clf, lossf, train_loader, val_loader, accuracy_score):
    # variables for early stopping
    n_epochs_stop = 10
    epochs_no_improve = 0
    min_val_loss = float('inf')

    # running epoch
    for epoch in range(1, epochs + 1):

        # variables for performance monitoring
        loss_train_list = []
        loss_val_list = []
        correct_train_list_fscroe = []
        correct_train_list = []
        correct_val_list_fscroe = []
        correct_val_list = []

        # training on batches
        for (text, lens, mask, labels), (val_text, val_lens, val_mask, val_labels) in zip(train_loader, val_loader):
            # - training section - #
            clf.train()
            text, lens, mask, labels = text.to(device),lens , mask.to(device), labels.to(device)
            output = clf(text, mask, lens)
            # compute loss function
            loss = lossf(output, labels)
            # append loss output to loss_train_list for mnitoring
            loss_train_list.append(loss.item())

            # convert tensors to numpy array
            labels = labels.cpu().detach().numpy()
            output = torch.argmax(output, dim=1).cpu().detach().numpy()
            # calculate accuracy score
            correct_train_list_fscroe.append(f1_score(output, labels, average='macro'))
            correct_train_list.append(accuracy_score(output, labels))
            # compute back propagation
            loss.backward()
            # update weights
            optim.zero_grad()
            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
            optim.step()
            # clean gradients to not accumulate

            # - evaluation section - #
            clf.eval()
            with torch.no_grad():
                # - training section - #
                val_text, val_lens, val_mask, val_labels = val_text.to(device), val_lens.to(device), val_mask.to(device), val_labels.to(device)
                val_output = clf(val_text, val_mask, val_lens).squeeze()

                # compute loss function
                val_loss = lossf(val_output, val_labels)
                # append loss output to loss_val_list for mnitoring
                loss_val_list.append(val_loss.item())

                # convert tensors to numpy array
                val_output = torch.argmax(val_output, dim=1).cpu().detach().numpy()
                val_labels = val_labels.cpu().detach().numpy()

                # calculate accuracy score
                correct_val_list.append(accuracy_score(val_output, val_labels))
                correct_val_list_fscroe.append(f1_score(val_output, val_labels, average='macro'))
              
            # change lr if the loss didn't decrease

            loss = torch.mean(torch.FloatTensor(loss_train_list))
            val_loss = torch.mean(torch.FloatTensor(loss_val_list))
            acc = torch.mean(torch.FloatTensor(correct_train_list)) * 100
            fscore = torch.mean(torch.FloatTensor(correct_train_list_fscroe)) * 100
            val_acc = torch.mean(torch.FloatTensor(correct_val_list)) * 100
            val_fscore = torch.mean(torch.FloatTensor(correct_val_list_fscroe)) * 100

        scheduler.step(val_loss)

        # save best model if the loss is the best
        if torch.mean(torch.FloatTensor(loss_val_list)) < min_val_loss:
          # variables for best performance
          best_epoch = epoch
          best_loss = loss
          best_val_loss = val_loss
          best_acc = acc
          best_fscore = fscore
          best_val_acc = val_acc
          best_val_fscore = val_fscore
          # save best model
          epochs_no_improve = 0
          min_val_loss = best_val_loss
          # print the current epoch as the best epoch

          print(
              f'BEST EPOCH: Epoch({epoch}) -> Train: (Accuracy: {best_acc:.1f}, f-score: {best_fscore:.1f}, Loss: {best_loss:.4f}) | Val: (Accuracy: {best_val_acc:.1f}, f-score: {best_val_fscore:.1f}, Loss: {best_val_loss:.4f})')

        else:
            # print the current epoch as normal epoch
            print(
                f'Epoch({epoch}) -> Train: (Accuracy: {acc:.1f}, f-score: {fscore:.1f}, Loss: {loss:.4f}) | Val: (Accuracy: {val_acc:.1f}, f-score: {val_fscore:.1f},Loss: {val_loss:.4f})')
            # if epochs_no_improve reached n_epochs_stop the training will stop
            epochs_no_improve += 1

        # early stop the training
        if epoch > 5 and epochs_no_improve == n_epochs_stop:
            torch.save(clf, f'clf_val_loss_{best_val_loss:.4f}_f-score_{best_val_fscore:.1f}_val_acc_{best_val_acc:.1f}.pt')
            print('Early stopping!')
            print()
            print(
                f'BEST EPOCH: Epoch({best_epoch}) -> Train: (Accuracy: {best_acc:.1f}, f-score: {best_fscore:.1f}, Loss: {best_loss:.4f}) | Val: (Accuracy: {best_val_acc:.1f}, f-score: {best_val_fscore:.1f}, Loss: {best_val_loss:.4f})')
            break

print('Running...')
big_training_loop(555, optimizer, model, cross_entropy, train_loader, val_loader, accuracy_score=accuracy_score)

Running...
BEST EPOCH: Epoch(1) -> Train: (Accuracy: 28.4, f-score: 25.2, Loss: -0.0608) | Val: (Accuracy: 31.3, f-score: 23.5, Loss: -0.0249)
Epoch(2) -> Train: (Accuracy: 28.0, f-score: 25.4, Loss: -0.0458) | Val: (Accuracy: 31.3, f-score: 23.5,Loss: -0.0249)
Epoch(3) -> Train: (Accuracy: 32.5, f-score: 29.5, Loss: -0.0696) | Val: (Accuracy: 31.3, f-score: 23.5,Loss: -0.0249)
Epoch(4) -> Train: (Accuracy: 36.0, f-score: 33.0, Loss: -0.0618) | Val: (Accuracy: 31.3, f-score: 23.5,Loss: -0.0249)
Epoch(5) -> Train: (Accuracy: 31.1, f-score: 27.0, Loss: -0.0599) | Val: (Accuracy: 31.3, f-score: 23.5,Loss: -0.0249)
Epoch(6) -> Train: (Accuracy: 35.1, f-score: 31.0, Loss: -0.0712) | Val: (Accuracy: 31.3, f-score: 23.5,Loss: -0.0249)
Epoch(7) -> Train: (Accuracy: 34.0, f-score: 31.3, Loss: -0.0641) | Val: (Accuracy: 31.3, f-score: 23.5,Loss: -0.0249)
Epoch(8) -> Train: (Accuracy: 34.7, f-score: 30.9, Loss: -0.0722) | Val: (Accuracy: 31.3, f-score: 23.5,Loss: -0.0249)
Epoch(9) -> Train: (Accuracy: 31.8, f-score: 28.1, Loss: -0.0570) | Val: (Accuracy: 31.3, f-score: 23.5,Loss: -0.0249)
Epoch(10) -> Train: (Accuracy: 35.3, f-score: 31.3, Loss: -0.0598) | Val: (Accuracy: 31.3, f-score: 23.5,Loss: -0.0249)
Epoch(11) -> Train: (Accuracy: 38.0, f-score: 33.8, Loss: -0.0860) | Val: (Accuracy: 31.3, f-score: 23.5,Loss: -0.0249)
Early stopping!

BEST EPOCH: Epoch(1) -> Train: (Accuracy: 28.4, f-score: 25.2, Loss: -0.0608) | Val: (Accuracy: 31.3, f-score: 23.5, Loss: -0.0249)

I don’t know why the accuracy is so Low; I tried  many loss functions.
The way bert is implemented is inspired from this tutorial 4