I am trying to classify (2 classes) images in my dataset.
First of all, my datasets were highly imbalanced (0.97 against 0.03) and when I tried to train network, test showed me the same numbers for every item ([~0.97 for first class, ~0.03 for the second class]). Loss was around 0.6 every epoch, no matter of learning rate (from 1e-5 to 3).
Then I balanced my classes (50/50) but loss stayed exactly the same! Around 0.6 every epoch, no matter of learning rate (from 1e-5 to 3). Results on the test became also [~0.5 for first class, ~0.5 for the second class]).
Then I tried to train second network (first was convolutional one), this was fc. Results were EXACTLY the same.
What can be problem here?
Here is the code:
class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(1, 30, 5, 2)
        self.conv2 = nn.Conv2d(30, 50, 5, 1)
        # self.conv3 = nn.Conv2d(30, 50, 5, 1)
        self.dropout = nn.Dropout2d()
        self.fc1 = nn.Linear(5*5*50, 100)
        self.fc2 = nn.Linear(100, 2)
        

    def forward(self, x):
        x = F.relu(self.conv1(x))
        
        x = F.max_pool2d(x, 2, 2)
        x = F.relu(self.conv2(x))
        x = F.max_pool2d(x, 2, 2)
        #print(x.shape)
        #x = F.relu(self.conv3(x))
        #x = F.max_pool2d(x, 2, 2)
        # print(x.shape)
        x = x.view(-1, 5*5*50)
        x = F.relu(self.fc1(x))
        # x = F.dropout(x, training=self.training)
        x = self.fc2(x)
        #print(x.data.max(), x.data.min() )
        return F.sigmoid(x)

class Net2(nn.Module):
    def __init__(self):
        super(Net2, self).__init__()       
        self.fc1 = nn.Linear(64*64, 8*8)
        self.fc2 = nn.Linear(64, 2)
        

    def forward(self, x):
        x = x.view(-1, 64*64)
        x = F.relu(self.fc1(x))
        # x = F.dropout(x, training=self.training)
        x = self.fc2(x)
        #print(x.data.max(), x.data.min() )
        return F.sigmoid(x)

net = Net2().cuda()

class PillDataset(Dataset):
    def __init__(self):
        self.X = []
        self.y = []
        self.transform = transforms.Compose([transforms.RandomAffine(degrees=[-40,40],
                                                                    translate = [0.1,0.1]),
                                            
                                            transforms.RandomHorizontalFlip(p=0.6),
                                             transforms.RandomVerticalFlip(p=0.5),
                                             transforms.Scale(64),
                                             transforms.ToTensor()])

        filenames = os.listdir('./correct/')
        filenames = np.random.choice(filenames, 1000)
        for fn in filenames:
            self.X.append('./correct/'+fn)
            #self.y.append([1, 0])
            self.y.append(1)
        filenames = os.listdir('./defect/')
        for fn in filenames:
            self.X.append('./defect/'+fn)
            #self.y.append([0, 1])
            self.y.append(2)
    def __getitem__(self, index):
        image = Image.open(self.X[index])
        # image = cv2.Laplacian(image,cv2.CV_64F)
        image = self.transform(image)
        label = torch.tensor(self.y[index])
        return image, label

    def __len__(self):
        return len(self.X)


full_dataset = PillDataset()
train_size = int(0.8 * len(full_dataset))
test_size = len(full_dataset) - train_size
train_set, test_set = torch.utils.data.dataset.random_split(full_dataset, [train_size, test_size])

train_loader = DataLoader(train_set,
                          batch_size=256,
                          shuffle=True,
                          num_workers=0,
                         pin_memory=True 
                         )

optimizer = optim.SGD(net.parameters(), lr=1e-3, momentum=0.9)
criterion = nn.CrossEntropyLoss()

def train(epoch):
    running_loss = 0.0
    net.train()
    
    for batch_idx, (data, target) in enumerate(train_loader):
        data, target = data.cuda(async=True), target.cuda(async=True)
        
        optimizer.zero_grad()
        output = net(data)

        loss = criterion(output, target)
        loss.backward()
        optimizer.step()
        running_loss += loss.item()
        if batch_idx == 0:
            print('Train Epoch: {} [{}]'.format(epoch, running_loss))
            running_loss = 0.0

for epoch in range(1, 30):
    train(epoch)