Yes you have to write a custom loss function but you could simply do it like this:
class CustomLoss(torch.nn.Module):
    def __init__(self, loss_fn=MSELoss):
        super().__init__(self)
        self.loss_fn = loss_fn()

    def forward(self, pred_list, target_list):
        assert len(pred_list) == len(target_list)
        value = 0
        for _pred, _target in zip(pred_list, target_list):
            value = value + self.loss_fn(_pred, _target)

        return value