Do you want to overwrite the old values after each update?
Would this work?
model = nn.Sequential(
    nn.Linear(20, 10),
    nn.ReLU(),
    nn.Linear(10, 2)
)

optimizer = optim.SGD(model.parameters(), lr=1e-3)

x = torch.randn(1, 20)

# Store just the first layer
weights = dict(model[0].named_parameters())

for epoch in range(10):
    output = model(x)
    output.mean().backward()
    optimizer.step()
    # Update weights
    weights = dict(model[0].named_parameters())