I tried to print the loss before and after the optimize.step() and it didnt change.
As for the code, here it is:
optimizer = optim.Adam(net.parameters(), lr=0.1) 
net = conv_Net()
device = 'cuda' if torch.cuda.is_available() else 'cpu'
net = net.to(device)
net = torch.nn.DataParallel(net, device_ids=range(torch.cuda.device_count()))
cudnn.benchmark = True
loss_function=nn.CrossEntropyLoss()
for batch in trainloader:
        images, labels = batch
        images=images.to(device)
        labels=labels.to(device)
        optimizer.zero_grad() 
        preds = net(images)
        loss = loss_function(preds,labels)
        loss.backward() 
        optimizer.step() 
        total_loss += loss.item()
        total_correct += get_correct(preds, labels)