@albanD  I tried that , that is, using double tensor and it ran successfully on the train_set but giving the same error for the validation set which is being loaded from the same dataset class and I have also kept the model same for it.Actually I have written the train and validation set loss computation under the same function using the same model.
model = model.double()
def train(model, trainloader, validloader, optimizer, num_epochs=5):
    i=0;
    for epoch in tqdm(range(num_epochs)):
        i+=1
        train_loss_history = []
        valid_loss_history = []
        print("Epoch :", i)
        for images, targets, image_ids in tqdm(trainloader):
            images = list((torch.DoubleTensor(img.T)).to(device) for img in images)
            targets = [{k: v.to(device) for k, v in t.items()} for t in targets]

            train_loss_dict = model(images, targets)

            train_losses = sum(loss for loss in train_loss_dict.values())

            train_loss_history.append(train_losses)
            optimizer.zero_grad()
            train_losses.backward()
            optimizer.step()

        plt.plot(np.arange(len(train_loss_history)), train_loss_history)

        for images, targets, image_ids in tqdm(validloader):
            images = list((torch.DoubleTensor(img.T)).to(device) for img in images)
            targets = [{k: v.to(device) for k, v in t.items()} for t in targets]

            valid_loss_dict = model(images, targets)

            valid_losses = sum(loss for loss in valid_loss_dict.values())

            valid_loss_history.append(valid_losses)

        plt.plot(np.arange(len(valid_loss_history)), valid_loss_history)
        

    return model

If you want I can also share the my google colab link. Thanks in advance