Could you try to parse only the learnable parameters to the optimizer?
I think the problem is that even if grad param is set to false, it only means that values aren’t gonna be updated, but gradients are still computed as they have to be backpropagated to previous tensors.
import torch
import time
x = torch.rand(10,requires_grad=False)
y = torch.rand(10,requires_grad=False)
z = torch.rand(10,requires_grad=True)
def timing(f):
    def wrap(*args):
        time1 = time.time()
        ret = f(*args)
        time2 = time.time()
        print('{:s} function took {:.3f} ms'.format(f.__name__, (time2-time1)*1000.0))

        return ret
    return wrap
@timing
def run(optim,backprop=True):

    optim.zero_grad()
    q=x+y
    p=(z+q).mean()
    print(x)
    print(y)
    print(z)
    print(q)
    if backprop:
        p.backward()
    print(x.grad)
    print(y.grad)
    print(z.grad)
    print(q.grad)
    print(p.grad)
    if backprop:
        optim.step()
    print(x)
    print(y)
    print(z)
    print(q)

run( torch.optim.SGD([x, y, z], lr=1))
with torch.no_grad():
    run( torch.optim.SGD([x, y, z], lr=1),False)
run( torch.optim.SGD([x], lr=1))

tensor([0.8253, 0.7537, 0.7688, 0.9117, 0.2731, 0.4141, 0.5021, 0.5691, 0.8326,
        0.9245])
tensor([0.2086, 0.2583, 0.3304, 0.6977, 0.2884, 0.6722, 0.2859, 0.4168, 0.3863,
        0.2488])
tensor([0.0729, 0.0993, 0.8631, 0.5255, 0.8684, 0.8381, 0.3063, 0.4755, 0.3240,
        0.5715], requires_grad=True)
tensor([1.0340, 1.0120, 1.0993, 1.6094, 0.5615, 1.0863, 0.7880, 0.9859, 1.2189,
        1.1733])
None
None
tensor([0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000,
        0.1000])
None
None
tensor([0.8253, 0.7537, 0.7688, 0.9117, 0.2731, 0.4141, 0.5021, 0.5691, 0.8326,
        0.9245])
tensor([0.2086, 0.2583, 0.3304, 0.6977, 0.2884, 0.6722, 0.2859, 0.4168, 0.3863,
        0.2488])
tensor([-2.7121e-02, -7.1023e-04,  7.6311e-01,  4.2552e-01,  7.6838e-01,
         7.3814e-01,  2.0632e-01,  3.7555e-01,  2.2402e-01,  4.7150e-01],
       requires_grad=True)
tensor([1.0340, 1.0120, 1.0993, 1.6094, 0.5615, 1.0863, 0.7880, 0.9859, 1.2189,
        1.1733])
run function took 62.251 ms
tensor([0.8253, 0.7537, 0.7688, 0.9117, 0.2731, 0.4141, 0.5021, 0.5691, 0.8326,
        0.9245])
tensor([0.2086, 0.2583, 0.3304, 0.6977, 0.2884, 0.6722, 0.2859, 0.4168, 0.3863,
        0.2488])
tensor([-2.7121e-02, -7.1023e-04,  7.6311e-01,  4.2552e-01,  7.6838e-01,
         7.3814e-01,  2.0632e-01,  3.7555e-01,  2.2402e-01,  4.7150e-01],
       requires_grad=True)
tensor([1.0340, 1.0120, 1.0993, 1.6094, 0.5615, 1.0863, 0.7880, 0.9859, 1.2189,
        1.1733])
None
None
tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0.])
None
None
tensor([0.8253, 0.7537, 0.7688, 0.9117, 0.2731, 0.4141, 0.5021, 0.5691, 0.8326,
        0.9245])
tensor([0.2086, 0.2583, 0.3304, 0.6977, 0.2884, 0.6722, 0.2859, 0.4168, 0.3863,
        0.2488])
tensor([-2.7121e-02, -7.1023e-04,  7.6311e-01,  4.2552e-01,  7.6838e-01,
         7.3814e-01,  2.0632e-01,  3.7555e-01,  2.2402e-01,  4.7150e-01],
       requires_grad=True)
tensor([1.0340, 1.0120, 1.0993, 1.6094, 0.5615, 1.0863, 0.7880, 0.9859, 1.2189,
        1.1733])
run function took 1.616 ms
tensor([0.8253, 0.7537, 0.7688, 0.9117, 0.2731, 0.4141, 0.5021, 0.5691, 0.8326,
        0.9245])
tensor([0.2086, 0.2583, 0.3304, 0.6977, 0.2884, 0.6722, 0.2859, 0.4168, 0.3863,
        0.2488])
tensor([-2.7121e-02, -7.1023e-04,  7.6311e-01,  4.2552e-01,  7.6838e-01,
         7.3814e-01,  2.0632e-01,  3.7555e-01,  2.2402e-01,  4.7150e-01],
       requires_grad=True)
tensor([1.0340, 1.0120, 1.0993, 1.6094, 0.5615, 1.0863, 0.7880, 0.9859, 1.2189,
        1.1733])
None
None
tensor([0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000,
        0.1000])
None
None
tensor([0.8253, 0.7537, 0.7688, 0.9117, 0.2731, 0.4141, 0.5021, 0.5691, 0.8326,
        0.9245])
tensor([0.2086, 0.2583, 0.3304, 0.6977, 0.2884, 0.6722, 0.2859, 0.4168, 0.3863,
        0.2488])
tensor([-2.7121e-02, -7.1023e-04,  7.6311e-01,  4.2552e-01,  7.6838e-01,
         7.3814e-01,  2.0632e-01,  3.7555e-01,  2.2402e-01,  4.7150e-01],
       requires_grad=True)
tensor([1.0340, 1.0120, 1.0993, 1.6094, 0.5615, 1.0863, 0.7880, 0.9859, 1.2189,
        1.1733])
run function took 1.965 ms

If you look at this messy code, even if grads are false, the fact you are running the optimizer takes lot of time meanwhile if you go for a no_grad strategy or directly does not parse them to the optimizer you save that time.