I have a convolutional neural network in vgg architecture “style” (down below) to classify images if there is a cat on the picture, or a dog. My training set contains 25000 images cropped to 256px each side. I tried different learning rates, different loss functions and much more but my loss keeps fluctuating between 0.692 and 0.694, but it will not decrease…
    normalize = transforms.Normalize(
        mean=[0.485, 0.456, 0.406],
        std=[0.229, 0.224, 0.225]
    )

    transform = transforms.Compose([
        transforms.Resize(256),
        transforms.CenterCrop(256),
        transforms.ToTensor(),
        normalize
    ])

    # Output = [isDog, isCat]
    train_data_list = []
    train_data = []
    target_list = []
    plotlist = []
    train_files = listdir("data/catsdogs/train/")


    def loadTrainData():
        global train_data_list
        global train_data
        global target_list
        print("Loading data now...")

        amount = len(listdir("data/catsdogs/train/"))
        current = 0

        for i in range(amount):
            r = random.randint(0, len(train_files) - 1)
            file = train_files[r]
            train_files.remove(file)

            img = Image.open("data/catsdogs/train/" + file)
            img_tensor = transform(img)  # (3, 256, 256)

            isCat = 1 if 'cat' in file else 0
            isDog = 1 if 'dog' in file else 0
            target = [isCat, isDog]

            train_data_list.append(img_tensor)
            target_list.append(target)

            if len(train_data_list) >= 64:
                train_data.append((torch.stack(train_data_list), target_list))
                train_data_list = []
                target_list = []

            current = current + 1
            print("Loaded: {:.1f}%".format(current * 100 / amount))
        print("Loaded data successfully!")


    class Network(nn.Module):
        def __init__(self):
            super(Network, self).__init__()
            self.conv1 = nn.Conv2d(3, 16, kernel_size=3, stride=1, padding=1)
            self.conv2 = nn.Conv2d(16, 16, kernel_size=3, stride=1, padding=1)

            self.conv3 = nn.Conv2d(16, 32, kernel_size=3, stride=1, padding=1)
            self.conv4 = nn.Conv2d(32, 32, kernel_size=3, stride=1, padding=1)

            self.conv5 = nn.Conv2d(32, 64, kernel_size=3, stride=1, padding=1)
            self.conv6 = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1)

            self.dropout = nn.Dropout2d()
            self.relu = nn.ReLU(inplace=True)
            self.pool = nn.MaxPool2d(kernel_size=3, stride=3, padding=0, dilation=1, ceil_mode=False)

            self.fc1 = nn.Linear(5184, 1296)
            self.fc2 = nn.Linear(1296, 2)

        def forward(self, x):
            # Block 1
            x = self.conv1(x)
            x = self.relu(x)

            x = self.conv2(x)
            x = self.relu(x)

            x = self.pool(x)

            # Block 2
            x = self.conv3(x)
            x = self.relu(x)

            x = self.conv4(x)
            x = self.relu(x)

            x = self.pool(x)

            # Block 3
            x = self.conv5(x)
            x = self.relu(x)

            x = self.conv6(x)
            x = self.relu(x)

            x = self.pool(x)

            x = x.view(-1, 5184)

            x = F.relu(self.fc1(x))
            x = self.dropout(x)
            x = self.fc2(x)

            return torch.sigmoid(x)


    model = Network()
    model = model.cuda()

    optimizer = optim.SGD(model.parameters(), lr=0.0001, weight_decay=0.0016)


    def train(epoch):
        global optimizer

        model.train()
        batch_id = 0
        for data, target in train_data:
            data = data.cuda()
            target = torch.Tensor(target).cuda()

            data = Variable(data)
            target = Variable(target)

            optimizer.zero_grad()

            out = model(data)
            criterion = F.binary_cross_entropy

            loss = criterion(out, target)
            loss.backward()

            optimizer.step()

            plotlist.append(loss)

            print('Train Epoch: {},  {:.0f}% ,\tLoss: {:.6f}'.format(
                epoch, 100. * batch_id / len(train_data), loss.item()
            ))
            batch_id = batch_id + 1


    loadTrainData()

    for epoch in range(25):
        train(epoch)

    plt.plot(plotlist)
    plt.show()
    plt.ylabel("Loss")
    plt.savefig("lossPlot.png")


If I increase the learning rate, the fluctuation get heavier… For example for lr = 0.1 between 0.5 and 0.7
Also here is a plot of my loss over 5 iterations with lr 0.0001: