Coooool! That works.
here’s my testing code
class myModel(torch.nn.Module):
    def __init__(self):
        super(myModel, self).__init__()
        self.filter = torch.nn.Parameter(torch.randn(8, 4, 3, 3))

    def forward(self, x):
        return torch.nn.functional.conv2d(x, self.filter)

inputs = torch.randn(1, 4, 5, 5).cuda()
model = myModel().cuda()

gt = torch.randn(1, 8, 3, 3).cuda()
optimizer = torch.optim.SGD(model.parameters(), lr=0.01, momentum=0.9)

for _ in range(10):
    optimizer.zero_grad()
    result = model(inputs)
    loss = torch.nn.functional.mse_loss(result, gt)
    print(loss)
    loss.backward(retain_graph=True)
    optimizer.step()