Here is a code snippet of my idea:
# Setup
lin = nn.Linear(5, 5, bias=False)
weight = nn.Parameter(torch.randn(5, 5))
optimizer = torch.optim.SGD([weight], lr=1.)

# Freeze lin.weight
lin.weight.requires_grad = False

# Forward pass
x = torch.randn(1, 5)
out = F.linear(x, lin.weight + weight)

# Backward pass
out.mean().backward()

# Check grads
print(lin.weight.grad)
> None

print(weight.grad)
> tensor([[-0.0892, -0.3502, -0.0987, -0.3553, -0.2443],
          [-0.0892, -0.3502, -0.0987, -0.3553, -0.2443],
          [-0.0892, -0.3502, -0.0987, -0.3553, -0.2443],
          [-0.0892, -0.3502, -0.0987, -0.3553, -0.2443],
          [-0.0892, -0.3502, -0.0987, -0.3553, -0.2443]])

# Step
optimizer.step()

Let me know, if this would work for you.