Sorry, if I didn’t make it clear. I do am using the I am using the synchronize functionality, I just didn’t wanted the code to look like spaghetti. I insert it in between every line shown as follows. I also use an Averager object to access the steady state averages of each time measure.
#Initialize averager objects to accumulate time measures
batch_time = Averager()
data_time = Averager()
cuda_time = Averager()
grad_time = Averager()
forward_time =Averager()
loss_time  = Averager()
prediction_time = Averager()
accuracy_time = Averager()
optimizer_time = Averager()
backward_time = Averager()
#Initialize per epoch timer
end_time = time.time()
for data in dataloader:
    #data loading
    data_time.update(time.time() - end_time)
    #load gpu
    data = data.cuda(async=False)
    torch.cuda.synchronize()
    cuda_time.update(time.time() - end_time)
    #set gradient
    data = torch.torch.autograd(data, requires_grad=True)
    torch.cuda.synchronize()
    grad_time.update(time.time() - end_time)
    #forward call
    output = model(data)
    torch.cuda.synchronize()
    forward_time.update(time.time() - end_time)
    #compute loss
    loss  = criterion(output, labels)
    torch.cuda.synchronize()
    loss_time.update(time.time() - end_time)
    #make predictions
    predictions = (output.detach() > 0).int()
    torch.cuda.synchronize()
    prediction_time.update(time.time() - end_time)
    #calculate accuracy
    accuracy = (labels.detach().int() == predictions).sum().item()/batch_size
    torch.cuda.synchronize()
    accuracy_time.update(time.time() - end_time)
    #optimize
    optimizer.zero_grad()
    torch.cuda.synchronize()
    optimizer_time.update(time.time() - end_time)
    loss.backward() ### backward takes 100x longer than forward in torch 1.4.0
    torch.cuda.synchronize()
    backward_time.update(time.time() - end_time)
    optimizer.step()
    torch.cuda.synchronize()
    batch_time.update(time.time() - end_time)
    #restart the timer
    end_time = time.time()

#Then print the average measures for the epoch
print(
    Time ({batch_time.avg:.3f})\t'
    'Backward  ({backward_time.avg:.3f})\t'
    'Optimizer  ({optimizer_time.avg:.3f})\t'
    'Accuracy  ({accuracy_time.avg:.3f})\t'
    'Loss  ({loss_time.avg:3f})\t'
    'Forward  ({forward_time.avg:3f})\t'
    'Grad  ({grad_time.avg:3f})\t'
    'Cuda  ({cuda_time.avg:.3f})\t'
    'Data  ({data_time.avg:.3f})\t''.format(
    backward_time=backward_time,
    optimizer_time=optimizer_time,
    forward_time=forward_time,
    accuracy_time=accuracy_time,
    loss_time=loss_time,
    grad_time=grad_time,
    batch_time=batch_time,
    cuda_time=cuda_time,
    data_time=data_time))

These are my recorded measures when running on torch 1.4.0:
Time (16.885),   Backward (16.885), Optimizer (0.476), Accuracy (0.476),  Loss (0.476), Forward (0.457),  Grad (0.348), Cuda (0.348),  Data (0.277)
Then when run on the same machine using torch 0.4.1:
Time (0.524) , Backward (0.520),  Optimizer (0.510), Accuracy (0.510),  Loss (0.508), Forward (0.419565), Grad (0.405319), Cuda (0.405), Data (0.229)