So well, now I changed my code to this (Commented out the manual weight update and used the optimizer object instead):
import numpy as np
import pandas as pd
import torch
import matplotlib.pyplot as plt
import torchvision
import csv 
from torch.autograd import Variable
import glob
import torch.optim as optim
import torch.nn as nn
import torch.nn.functional as F
from random import shuffle 
import glob
import os
from data_loader import CatDogDataset
from torch.utils.data import DataLoader
from tqdm import tqdm
from torchsummary import summary
import sys

class Net(nn.Module):
    def __init__(self):
        super(Net,self).__init__()
        self.conv1 = nn.Conv2d(3,base_filters,3, padding = 1) 
        self.pool1 = nn.MaxPool2d(2)
        
        self.conv2 = nn.Conv2d(base_filters,base_filters*2,3, padding = 1)
        self.pool2 = nn.MaxPool2d(2)
        
        self.conv3 = nn.Conv2d(base_filters*2,base_filters*4,3, padding = 1)
        self.pool3 = nn.MaxPool2d(2)
        
        self.conv4 = nn.Conv2d(base_filters*4,base_filters*8,3, padding = 1)
        self.pool4 = nn.MaxPool2d(2)
        
        self.fc1 = nn.Linear(base_filters*8*8*8,120)
        self.fc2 = nn.Linear(120,100)
        self.fc3 = nn.Linear(100,1)
            
        
    def forward(self,x):
        x  = self.pool1(F.relu(self.conv1(x)))
        x  = self.pool2(F.relu(self.conv2(x)))
        x  = self.pool3(F.relu(self.conv3(x)))
        x  = self.pool4(F.relu(self.conv4(x)))
 
        x = x.view(-1,base_filters*8*8*8)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        #x = F.softmax(x)
        return x

base_filters = 50
batch_size = 1
net = Net()

loss_fn = torch.nn.MSELoss()
dataset_train = CatDogDataset("/home/megh/work/catdog/CD_data.csv","/home/megh/work/catdog/")
train_loader = DataLoader(dataset_train, batch_size = batch_size, shuffle= True, num_workers=1)

optimizer = optim.Adam(net.parameters(), lr = 0.1)
net.cuda()

summary(net, input_size=(3, 128, 128))

net.train()
tmp = []
lr = 1
for batch_idx, (animal) in enumerate(train_loader):      
    image = animal['image_data']
    image = image.cuda()
    gt = animal['gt']
    gt = gt.cuda()
    image, gt = Variable(image, requires_grad = True), Variable(gt, requires_grad = True)
    output = net(image)
    loss =  loss_fn(output,gt)
    loss.backward()
    optimizer.step()
    optimizer.zero_grad()

# =============================================================================
#     for p in net.parameters():
#         p.sub_(lr* p.grad)
#         p.grad.zero_()
# =============================================================================
    print(loss)
    tmp.append(net.parameters())


I ran this for about 141 iterations i.e 141 images and ran the following small code snippet to check if the weights of the parameters are updating at all :
for p,q in zip(tmp[0],tmp[140]):
    print(p==q)


Here tmp is the list-element which stores the model parameters every iteration (given in code)
The output that it gives is a series of tensors with all elements being equal to 1
Please let me know why the optimizer isn’t updating the weights. (I’ve spent like 24 hours trying to debug this )
Thanks a lot!