Could you share a reproducible snippet? I could not reproduce this with
>>> import torch
>>> input = torch.randn(1000, 1000)
>>> input.requires_grad = True
>>> weight = torch.randn(1000, 1000)
>>> weight.requires_grad = True
>>> output = input * weight
>>> torch.autograd.grad(output.mean(), input)
(tensor([[ 3.6471e-07, -5.4263e-07,  3.5560e-08,  ..., -2.5698e-08,
         -4.6246e-07, -1.2394e-06],
        [-2.9459e-07,  2.4342e-06,  9.5083e-07,  ...,  2.1179e-06,
          4.3364e-07,  9.8099e-07],
        [ 3.5879e-07, -2.9793e-07, -3.0315e-08,  ..., -8.2863e-07,
         -3.8005e-07,  9.4537e-07],
        ...,
        [ 9.3157e-07, -4.0749e-08, -9.6082e-07,  ...,  9.7867e-07,
         -5.8472e-07,  8.4605e-07],
        [ 1.3695e-06, -3.3069e-07, -1.4985e-07,  ..., -2.6728e-07,
         -1.3704e-07,  5.8181e-07],
        [ 7.2464e-07, -4.6793e-07, -4.7812e-07,  ...,  7.8470e-08,
          1.1135e-06, -3.6685e-08]]),)
>>>