I modify my code and and wrote it using torch.utils.benchmark with warmup iterations, however, I still see the same behavior, using autograd is much slower than calculating gradient manually, and the difference depends on the gpu you use:
my code:
import timeit
import time
from torch.autograd import grad
import torch.nn as nn
import torch
INPUT_SIZE = 256
HIDDEN_SIZE = 2048
BASIS_SIZE = [INPUT_SIZE, HIDDEN_SIZE]
batch_size = 100
device_name = 'cuda:2'
device = torch.device(device_name)
BASIS_SIZE = [INPUT_SIZE, HIDDEN_SIZE]
X = torch.nn.Linear(BASIS_SIZE[1],BASIS_SIZE[0],bias=False).to(device)
target = torch.randn([batch_size, INPUT_SIZE]).to(device)

setup = '''import torch
import time
from torch.autograd import grad
import torch.nn as nn
INPUT_SIZE = 256
HIDDEN_SIZE = 2048
BASIS_SIZE = [INPUT_SIZE, HIDDEN_SIZE]
batch_size = 100
device_name = 'cuda:2'
device = torch.device(device_name)
eta=0.01

w = torch.FloatTensor(batch_size,HIDDEN_SIZE).fill_(0).to(device)
w.requires_grad = True'''

no_autograd='''
Res = -target+X(w)
loss = 0.5*(Res**2).sum()
w.data = w.data.add(-eta*X.weight.data.t().mm(Res.T).T)
'''
autograd='''
Res = -target+X(w)
loss = 0.5*(Res**2).sum()
w_grad = grad(outputs=loss, inputs=w)
w.data = w.data.add(-eta*w_grad[0].data)'''


t0 = timeit.Timer(
    stmt=no_autograd,
    setup=setup,
    globals={'X': X,'target':target})

t1 = timeit.Timer(
    stmt=autograd,
    setup=setup,
    globals={'X': X,'target':target})
num_iter = 2000
# warmUP
t0.timeit(num_iter)
noautograd_time = t0.timeit(num_iter) / num_iter * 1e6
print(f'noautograd:  {round(noautograd_time,2)} us')
# warmUP
t1.timeit(num_iter)
autograd_time = t1.timeit(num_iter) / num_iter * 1e6
print(f'autograd:      {round(autograd_time,2)} us')
print('Using autograd is {}x slower than without autograd'.format(round(autograd_time/noautograd_time,3)))

Result (on 1080Ti):
noautograd:  165.49 us
autograd:      592.85 us
Using autograd is 3.583x slower than without autograd

Result (on Tesla T4):
noautograd:  146.71 us
autograd:      401.26 us
Using autograd is 2.735x slower than without autograd