Note that even after fixing the optimizer issue, the weights should not be compared using references.
For example, this code snippet will report the “saved” weights being the same because both references refer to the same underlying tensor. Copying the weights before training will produce the expected results:
import torch

a = torch.nn.Linear(128,1)
b = torch.nn.Sigmoid()
optimizer = torch.optim.SGD(a.parameters(), 1e-3)
criterion = torch.nn.BCELoss()
saved = a.weight
detached = a.weight.detach().clone()
for i in range(0, 512):
    inp = torch.randn((1, 128))
    label = float(torch.sum(inp[64:]) > 0)
    label = torch.tensor([[label]])
    output = b(a(inp))
    loss = criterion(output, label)
    print(loss)
    loss.backward()
    optimizer.grad_ = None
    optimizer.step()
print(torch.allclose(saved, a.weight))
print(saved == a.weight)
print(torch.allclose(detached, a.weight))
print(detached == a.weight)