OK.
This is the command running the training.
from engine import train_one_epoch, evaluate

# '/content' is my google drive, i am using google colab with my drive mounted.
dataset = clothDataset('/content', get_transform(train=True))
dataset_test = clothDataset('/content', get_transform(train=False))

# split the dataset in train and test set
torch.manual_seed(1)
indices = torch.randperm(len(dataset)).tolist()
dataset = torch.utils.data.Subset(dataset, indices[:-50])
dataset_test = torch.utils.data.Subset(dataset_test, indices[-50:])


data_loader = torch.utils.data.DataLoader(
    dataset, batch_size=10, shuffle=True, num_workers=0,
    collate_fn=utils.collate_fn)

learning_scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
                                               step_size=3,
                                               gamma=0.1)
# I want to train it for 10 epochs
for epoch in range(10):
    train_one_epoch(model, optimizer, data_loader, device, epoch, print_freq=10)
    learning_scheduler .step()

I am suspecting that the dataset may be contributing to the problem. Please let me know if you need any information.