Hello,
I’m was making a neural network to to try a few option of pytorch, and then when I tried it on the classical Breast Cancer dataset my algo was just stuck. I simplified to the maximum my model, but accurracy seem to be blocked.
my model:
class Modela(torch.nn.Module):
    def __init__(self):
        super(Modela, self).__init__()
        self.l1 = torch.nn.Linear(30,16)
        self.l2 = torch.nn.Linear(16,4)
        self.l3 = torch.nn.Linear(4,2)
        
        self.sigmoid = torch.nn.Sigmoid()
        self.relu = torch.nn.ReLU()
        
    def forward(self, x):
        out1 = self.relu(self.l1(x)) 
        out2 = self.relu(self.l2(out1))
        y_pred = self.sigmoid(self.l3(out2))
        return y_pred

my training:
train_size = int(0.8 * len(full_dataset))
test_size = len(full_dataset) - train_size
train_dataset, test_dataset = torch.utils.data.random_split(full_dataset, [train_size, test_size])



model = Modela()
print(model)
#TRAIN
n_epoch = 50
criterion = torch.nn.CrossEntropyLoss()#torch.nn.BCELoss(reduction='sum')

        

        
def train(model,train_dataset,test_dataset,loss_fn,n_epoch=50):
    optimizer = torch.optim.Adam(model.parameters(), lr=0.0001)
    valid_losses = [] #list of the losses on valid
    valid_acc = [] #list of the accuracy on valid
    train_losses = [] #list of all the loss on training 
    train_loss_store = torch.zeros(1) #will store the current accuracy
    for e in range(n_epoch):
        print('n_epoch',e)
        for i in range(len(train_dataset)):

            #let's evaluate the performan,ce of my model
            if (i%100 == 0):

                score = torch.zeros(1)
                valid_loss_store = torch.zeros(1)
                
                with torch.no_grad():
                    for j in range(len(test_dataset)):
                        sample = test_dataset[j]
                        X, label = sample['X'], sample['label']
                        output = model(X)
                        output = torch.unsqueeze(output,0)
                        label = torch.unsqueeze(label,0)
                        loss = loss_fn(output, label)
                        valid_loss_store += loss
                        _, predicted = torch.max(output.data, 1)
                        score += predicted.eq(label).float()
                #let's print a few result
                score = score/len(test_dataset)
                valid_acc += [score.item()]
                valid_loss_store = valid_loss_store/len(test_dataset)
                valid_losses += [valid_loss_store.item()]
                train_loss_store = train_loss_store/100
                print('acc', score, 'valid loss :',valid_loss_store, 'train_loss_store', train_loss_store )
                train_losses += [train_loss_store.item()]
                train_loss_store = torch.zeros(1)
                    

   
            # here we train on the training dataset        
            sample = train_dataset[i]
            X, label = sample['X'], sample['label']
            optimizer.zero_grad()
            output = model(X)

            output = torch.unsqueeze(output,0)
            label = torch.unsqueeze(label,0)
            loss = loss_fn(output, label)
            train_loss_store += loss
            loss.backward()
#             for param in model.parameters():
#                 print(i,'gradient odg',param.grad.data.sum())
            optimizer.step()
    return train_losses, valid_losses, valid_acc
           
train_losses, valid_losses, valid_acc = train(model,train_dataset, test_dataset,criterion)

If I start this last part, sometimes I quickly grow to a 0.9 accurracy but most of the time I m just stuck to the exact first accuracy, with gradient close from zero. It looks like a local minimum maybe but in that case the fact of making a step for each data should enables it to escape this minimum. I don’t understand where it does come from.
I’ve tried to change the learning rate, and it does not change much.
Best regards
b