I have a model which consist of two sub-model, as follow,
import ModelA,ModelB
class model(nn.Module):
   def __init__(self, args):
     super(model,self).__init__()
      self.submodela = ModelA()
      self.submodelb = ModelB()
  def forward(self,x):
    x1 = self.submodela(x)
    x2 = self.submodelb(x1)

    return x1,x2

Model_A = ModelA()
Model_B = ModelA()
Model_All = model(args)
optimizer_a = optim.SGD(Model_A.parameters(), lr=0.01)
optimizer_b = optim.SGD(Model_B.parameters(), lr=0.01)
optimizer_all = optim.SGD(Model_All.parameters(), lr=0.01)

..................................................
...training...
...optimize as a whole...
optimizer_all.zero_grad()
outputs = Model_All(inputs)
loss= f(outputs)
loss.backward()
torch.nn.utils.clip_grad_norm(Model_All.parameters(),5)
optimizer_all.step()

........................................
...optimize as two parts...
optimizer_a.zero_grad()
optimizer_b.zero_grad()
x1 = Model_A(inputs)
outputs = Model_B(x1)
loss= f(outputs)
loss.backward()
torch.nn.utils.clip_grad_norm(Model_A.parameters(),5)
torch.nn.utils.clip_grad_norm(Model_B.parameters(),5)
optimizer_a.step()
optimizer_b.step()

…
The performance of these two optimization way is different, anybody can tell me why?