Thank you. This is how I want to add new blocks to the network dynamically. Can I use add_module as shown below?
class MyModel(nn.Module):
    def __init__(self):
        super(MyModel, self).__init__()
        self.modules = [convblock1, convblock2, convblock3]  # some list of modules
        self.convblock0 = self.conv(...)
        self.add_module("convblock1", self.modules[0]) # this doesn't look like the recommend way to use it 
        self.add_module("convblock2", self.modules[1])
        self.add_module("convblock3", self.modules[2])

    def forward(self,x, epoch_num):
        x = self.convblock0(x)
        if epoch_num >= 2:
            x = self.convblock1(x)
        if epoch_num >= 4:
            x = self.convblock2(x)
        if epoch_num >= 6:
            x = self.convblock3(x)
        return x

OR
class MyModel(nn.Module):
    def __init__(self):
        super(MyModel, self).__init__()
        self.modules = [convblock1, convblock2, convblock3]  # some list of modules
        self.convblock0 = self.conv(...)

    def forward(self,x, epoch_num):
        x = self.convblock0(x)
        if epoch_num == 2: # can I use add_module inside forward?
            self.add_module("convblock1", self.modules[0])
        if epoch_num >= 2:      
            x = self.convblock1(x)
        if epoch_num == 4:
            self.add_module("convblock2", self.modules[1])
        if epoch_num >= 4:
            x = self.convblock2(x)
        if epoch_num == 6:
            self.add_module("convblock3", self.modules[2])
        if epoch_num >= 6:
            x = self.convblock3(x)
        return x

Is this how I should update the optimizer after added new module to the model?
model = MyModel()
optimizer = torch.optim.Adam(model.convblock0.parameters(), learning_rate)
for epoch in range(10):
    if epoch == 2:
        new_par_dict = dict()
        new_par = model.modules[0].parameters()
        new_par_dict["params"] = new_par
        optimizer.add_param_group(new_par_dict)
    if epoch == 4:
        new_par_dict = dict()
        new_par = model.modules[1].parameters()
        new_par_dict["params"] = new_par
        optimizer.add_param_group(new_par_dict)
    if epoch == 6:
        new_par_dict = dict()
        new_par = model.modules[2].parameters()
        new_par_dict["params"] = new_par
        optimizer.add_param_group(new_par_dict)
    output = model(input, epoch)
    loss = loss_fun(output)
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()

But add_param_group expect dict. Is there a faster way to get param group dict of a specific layer? And also does this preserve previous state of the optimizer?
cc: @ptrblck