Thanks for pointing that out, I made the changes to the model:
class CustomModel(torch.nn.Module):
    def __init__(self, num_labels=3):
        super(CustomModel, self).__init__()
        self.base_model = BertModel.from_pretrained("bert-base-uncased") 
        self.lstm = torch.nn.LSTM(768, 64, bidirectional=True, batch_first=True)
        self.globalavgpooling = torch.nn.AdaptiveAvgPool1d(128) 
        self.globalmaxpooling = torch.nn.AdaptiveMaxPool1d(128)
        self.dropout = torch.nn.Dropout(p=0.3)
        self.linear = torch.nn.Linear(128, num_labels) 
        
    def forward(self, input_ids, attention_mask, labels):
        sequence_output, pooled_output = self.base_model(input_ids, attention_mask=attention_mask, return_dict=False)
        
        lstm_output, (h,c) = self.lstm(sequence_output) 
        avg_pool = self.globalavgpooling(lstm_output)
        max_pool = self.globalmaxpooling(lstm_output)
        
        concat = torch.cat((avg_pool, max_pool), dim=1)
        dropout = self.dropout(concat)
        
        output = self.linear(dropout[:, -1, :]) # Added [:, -1, :]
        
        # compile model 
        loss_fct = torch.nn.CrossEntropyLoss()
        loss = loss_fct(output, labels) 
        
        return loss, output

where I modified the outputs by using the last time step as you have suggested:

output = self.linear(dropout[:, -1, :]) # Added [:, -1, :]

after invoking batch_first=True in the LSTM layer.
Was this what you meant and implemented correctly?