I try to solve a simple Multivariate Linear Regression problem (argmin_w (Y-Xw)^2) using gradient descent.
The code using .backward() is 5 times lower than not using .backward():
import torch
import time

INPUT_SIZE = 256
HIDDEN_SIZE = 2048
BASIS_SIZE = [INPUT_SIZE, HIDDEN_SIZE]
batch_size = 100

X = torch.nn.Linear(BASIS_SIZE[1],BASIS_SIZE[0]).cuda()
target = torch.randn([batch_size, INPUT_SIZE]).cuda()
eta=0.01

w = torch.cuda.FloatTensor(batch_size,HIDDEN_SIZE).fill_(0)
w.requires_grad = True
start_time = time.time()
for k in range(2000):
    Res = X(w)-target
    loss = 0.5*(Res**2).sum()
    loss.backward()
    w.data = w.data.add(-eta*w.grad)
    X.weight.grad.zero_()
    w.grad.zero_()
print('With Backward -- Runtime: {}, loss: {}'.format(time.time()-start_time,loss))

w = torch.cuda.FloatTensor(batch_size,HIDDEN_SIZE).fill_(0)
w.requires_grad = True
start_time = time.time()
for k in range(2000):
    Res =X(w)-target
    loss = 0.5*(Res**2).sum()
    w.data = w.data.add(-eta*X.weight.data.t().mm(Res.T).T)
    
print('Without Backward -- Runtime: {}, loss: {}'.format(time.time()-start_time,loss))

With Backward -- Runtime: 1.8691670894622803, loss: 2.3017024993896484
Without Backward -- Runtime: 0.4285166263580322, loss: 2.3017029762268066