Hi, this is the simplest example of what I am trying to do in nn Dataparallell (this code hasn’t wrapped the net and othernet in DataParallel, but gives an idea of the logic that I am trying out)
class OtherNet():
   def _init_(self):
         self.layer1_other = nn.Linear(10,5)
   def forward(self,x):
         return self.layer1_other(x)


class Net():
   def _init_(self):
         self.layer1 = nn.Linear(10,5)
   def forward(self,x,other_net):
         y = some_function_of_other_net(x)
         return self.layer1(y)
def main():
   net = Net()
   other_net = OtherNet()
   scalar = 0
  #STEP 1.
   for data,target in dataset:
        output = net(data,other_net)
        loss = criterion(output,target)
        scalar = scalar + loss
   #got new scalar now run backprop on net
   #STEP 2.
   for data,target in dataset:
       loss.backward()
       optimizer.step()
  #Now use scalar to update other_net
  #STEP 3.
   for data,target in dataset:
         output = net(data,other_net)
         loss = criterion(output,target)
         other_net.param += some_function(scalar,loss)