class Net(nn.Module):
    def __init__(self, no_grad=False, requires_grad=True):
        super(Net, self).__init__()
        self.net_1 = Sequential(nn.Linear(1000,1000),
                  nn.Linear(1000,1000),
                  nn.Linear(1000,1000),
                  nn.Linear(1000,1000),
                  nn.Linear(1000,1000),
                  nn.Linear(1000,1000),)

        self.net_2 = nn.Linear(1000,1000)
        if not requires_grad:
            self._freeze_param()
        self.no_grad = no_grad
    
    def _freeze_param(self):
        for k,v in self.named_parameters():
            if k.startswith("net_1"):
                v.requires_grad = False
    
    def forward(self, x):
        if self.no_grad:
            with torch.no_grad():
                x = self.net_1(x)
        else:
            x = self.net_1(x)
        x = self.net_2(x)
        return x

def net_time_test(net, times):
    duration = 0
    start = time()
    for i in range(times):
        out = net(input_)
        start = time()
        out.backward(gradient=torch.randn(64, 1000))
        end = time()
        duration += end - start
    print(duration)

input_ = torch.randn(64, 1000)
net_baseline = Net(no_grad=False, requires_grad=True)
net_no_grad = Net(no_grad=True, requires_grad=True)
net_no_requires_grad = Net(no_grad=False, requires_grad=False)

print(net_time_test(net_baseline, 100))
# 21.8435652256
print(net_time_test(net_no_grad, 100))
# 9.42938923836
print(net_time_test(net_no_requires_grad, 100))
# 7.28591799736

In my experiment, requires_grad=False is faster than torch.no_grad() in training.