I think the following should work.
outputs.backward(torch.ones_like(outputs))
But my grads on the input tensor seem to stay None.
This is the code I am using.
net = Net()
net.load_state_dict(torch.load(PATH))

testset = torchvision.datasets.MNIST(root='./data',
                                     train=False,
                                     download=True,
                                     transform=transforms.ToTensor())
testloader = torch.utils.data.DataLoader(testset,
                                         batch_size=10000,
                                         shuffle=True)

optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)

correct = 0
total = 0
for data in testloader:
    images, labels = data
    outputs = net(images)
    _, predicted = torch.max(outputs.data, 1)
    total += labels.size(0)
    correct += (predicted == labels).sum().item()
    optimizer.zero_grad()
    outputs.backward(torch.ones_like(outputs), retain_graph=True)
    print(images.grad)