import torchvision, torch.nn as nn, torch, torchsummary

class CombineBase(nn.Module):
  def __init__(self, ModelOne, ModelTwo):
    super().__init__()
    self.modelone = ModelOne.to('cuda')
    self.modeltwo = ModelTwo.to('cuda')
    self.lin_one = nn.Linear(2000, 1000)
    self.lin_two = nn.Linear(1000, 10)
    self.softmax = nn.Softmax(dim=-1)
  def forward(self, x):
    out = torch.cat((self.modelone(x), self.modeltwo(x)), dim=-1)
    out = self.softmax(self.lin_two(self.lin_one(out)))
    return out

class CombineMiddle(nn.Module):
  def __init__(self, ModelOne, ModelTwo):
    super().__init__()
    self.modelone = ModelOne.to('cuda')
    self.modeltwo = ModelTwo.to('cuda')
    self.lin = nn.Linear(20, 10)
    self.softmax = nn.Softmax(dim=-1)
  def forward(self, x):
    out = torch.cat((self.modelone(x), self.modeltwo(x)), dim=-1)
    out = self.softmax(self.lin(out))
    return out

level_one = [torchvision.models.alexnet(pretrained=False) for i in range(16)]

def one_level(combination_type, number, level):
    return [combination_type(level[i], level[i+1]) for i in range(number) if i%2==0]

level_two = one_level(CombineBase, 16, level_one)
level_three = one_level(CombineMiddle, 8, level_two)
level_four = one_level(CombineMiddle, 4, level_three)
top_level = CombineMiddle(level_four[0], level_four[1])

torchsummary.summary(top_level, (3, 128, 128), batch_size=100)

would this ensure weights of each neural network get updated differently.
also it gives some error when using cuda, expected backend CPU got CUDA, for #4 ‘mat1’