Yes I am using the default VGG initializers:
def _initialize_weights(self):
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
                if m.bias is not None:
                    nn.init.constant_(m.bias, 0)
            elif isinstance(m, nn.BatchNorm2d):
                nn.init.constant_(m.weight, 1)
                nn.init.constant_(m.bias, 0)
            elif isinstance(m, nn.Linear):
                nn.init.normal_(m.weight, 0, 0.01)
                nn.init.constant_(m.bias, 0)

I am using a custom cross-entropy which is basically the same as the built-in function. I don’t think it is the reason though!
class Loss(torch.nn.Module):
    def __init__(self):
        super(Loss, self).__init__()
        self.epsilon = 1e-7
        self.log_softmax = nn.LogSoftmax(dim=1)
        self.softmax = nn.Softmax(dim=1)

    def forward(self, prediction, target):
        clamped = torch.clamp(-self.log_softmax(prediction), self.epsilon, 1000)
        loss = torch.mean((torch.sum(target.float() * clamped, dim=1)))
        return loss