Hi,
I am trying to find optimal hyperparameters for my deep learning model. I could able to optimize LR, BS and momentum. But I am strugging with optimization of number of layers and neurons within them. Could someone help me ?
Here is what I have written, what I expect is to modify this class to find optimal input_neurons and output_neurons within each layer.
class Net(nn.Module):
    def __init__(self, n_lay=2, neur=32, dropout=0.1):
        super().__init__()
        self.name = "DNN"
        self.init_args = {"n_lay": n_lay, "neur": neur}      
        self.fc = nn.ModuleList([nn.Linear(1105, neur)])    # 1105  features/descriptors
        self.fc.extend([nn.Linear(neur, neur) for i in range(1, n_lay-1)])
        self.fc.append(nn.Linear(neur, 1))
        self.dropout = nn.Dropout(p=dropout)

    def forward(self, x):
        for i, l in enumerate(self.fc):
            x = self.dropout(self.fc[i](x))
        return x

parameters:
opt_BO = [{"name": "lr", "type": "continuous", "domain": (0.00001, 0.001)},
          {"name": "bs", "type": "discrete", "domain": range(16, 512, 16)},
          {"name": "n_lay", "type": "discrete", "domain": range(2, 6)},
          {"name": "neur", "type": "discrete", "domain": range(8, 128, 8)},
          {"name": "dropout", "type": "discrete", "domain": np.linspace(0, 0.5, 11)}]

The f_opt function is as below:
def f_opt(parameters):
    parameters = parameters[0]
    # Data loading:
    trainloader = torch.utils.data.DataLoader(dataset_train, batch_size=int(parameters[1]), shuffle=True)
    testloader = torch.utils.data.DataLoader(dataset_val, batch_size=int(parameters[1]), shuffle=False)
    train_dl, valid_dl = (WrapDL(trainloader, to_gpu), WrapDL(testloader, to_gpu))    

    # Model definition:
    model = Net(
        n_lay=int(parameters[2]),
        neur=int(parameters[3]),
        dropout=parameters[4]
    )

    model.to(device)
    
    # Optimizer:
    opt = optim.SGD(model.parameters(), lr=parameters[0])
    scheduler = optim.lr_scheduler.CyclicLR(opt, parameters[0], parameters[0] * 3,
                                            step_size_up = int(len(train_dl) * int(parameters[1]) * epochs * 0.25))
    # Fit:
    score = fit(epochs, model, loss_func, scheduler, train_dl, valid_dl, calc_accuracy)
    
    return np.array(score)