So my implementation is to make a generator that spits out iteratively bigger and more complex networks. Since my amount of layers are dynamic I store them in a list bound to the class and I noticed that they aren’t automatically included in the .parameters, so I manually added them myself. After a round of testing the network almost immediately converges around 11-19% accuracy, so I have a strong feeling that the optimizer isn’t actually updating the layers. I’m having a hard time finding relevant examples on the google, can I get some advice?
class NNGenerator(torch.nn.Module):
  def __init__(self, num):
    super(NNGenerator, self).__init__()
    self.num = num
    self.res = 32
    self.power = 0
    self.layers = []

    for i in range(self.num * 2):
      self.layers.append( torch.nn.Conv2d( 3*(2**i), 3*(2**(i+1)), kernel_size=3) )
      self.res -= 2
      if(i % 2):
        self.res /= 2
        self.power = 3*(2**(i+1))
    
    self.pool = torch.nn.MaxPool2d(2, 2)

    self.feat = int( self.power * (self.res * self.res) )
    self.fc1 = torch.nn.Linear( self.feat, 100 )
    self.fc2 = torch.nn.Linear( 100, 10 )

  def forward(self, x): #-1, 3, 32, 32
    for idx, func in enumerate(self.layers):
      x = func(x)
      x = F.relu(x)
      if(idx % 2):
        x = self.pool(x) 
      
    x = F.relu( self.fc1(x.view(-1,self.feat)) )
    x = F.relu( self.fc2(x) )

    return F.softmax(x, -1)

teacher = NNGenerator(1)
#for idx, param in enumerate(teacher.layers):
#  teacher.register_parameter('conv'+str(idx), param)
#print(teacher.parameters)

optimizer = torch.optim.SGD(teacher.parameters(), lr=.001, momentum=0.9)
for param in teacher.layers:
  optimizer.add_param_group({'params': param.parameters()}) #aren't automatically added when they aren't a direct method
loss = torch.nn.CrossEntropyLoss()

def nntrain(epoch):
  teacher.train()

  for index, (images, targets) in enumerate(traingen):
    optimizer.zero_grad()
    output = teacher(images)
    if not (len(output) == len(targets)):
      print(len(output))
      print(len(targets))
    loss_out = loss(output, targets)
    loss_out.backward()
    optimizer.step()

    if not (index % 100):
      print(f'Epoch {epoch}: [{index*len(images)}/{len(trainin)}] Loss: {loss_out.item()}')

def nntest(epoch):
  teacher.eval() 

  test_loss = 0
  correct = 0

  with torch.no_grad():
    for images, targets in testgen:
      output = teacher(images)
      test_loss += loss(output, targets.long()).item()
      pred = output.data.max(1, keepdim=True)[1]
      correct += pred.eq(targets.data.view_as(pred)).sum()

  test_loss /= len(testin)
  print(f'Test result on epoch {epoch}: Avg loss is {test_loss}, Accuracy: {100.*correct/len(testin)}%')

for epoch in range(1, 3):
  nntrain(epoch)
  nntest(epoch)```