I believe the aux loss should not impact the weights of fc2 and should only impact fc1. However when I try to compare the weights of 2 networks one with aux and one without aux, the weights are different. I don’t fully understand why did that happen ?
class MyModel(nn.Module):
def init(self):
super(MyModel, self).init()
self.fc1 = nn.Linear(10, 64)
self.fc2 = nn.Linear(64, 10)
def forward(self, x):
    x1 = F.relu(self.fc1(x))
    x = self.fc2(x1)
    return x, x1

x = torch.randn(10, 10)
y = torch.randn(10, 10)
model = MyModel()
model2 = MyModel()
model2.load_state_dict(model.state_dict())
optimizer = torch.optim.SGD(model.parameters(), lr=1e-0)
optimizer2 = torch.optim.SGD(model.parameters(), lr=1e-0)
criterion = nn.MSELoss()
for epoch in range(100):
optimizer.zero_grad()
output, aux = model(x)
loss = criterion(output, y)
loss = loss + (aux**2).mean()
loss.backward()
optimizer.step()
optimizer2.zero_grad()
output2, _ = model2(x)
loss2 = criterion(output2, y)
loss2.backward()
optimizer2.step()

print((model2.fc2.weight.detach() == model.fc2.weight.detach()).all())