I have a balanced dataset -  - positive:negative = 1:1. I have been record the accuracy and loss of train and validation for each epoch, selected the best model based on validation accuracy, then plot the roc curve on train and test set. The final AUC doesn’t look bad, but the metrics vs epoch during training is strange. The two question I have is:

the first epoch have train accuracy as 0.8. I used pytorch’s default initiation and tried giving a uniformed distribution to sample the weights from. Both of them have larger than 0.75 starting accuracy on the training data. Is this normal?
The validation performance reach its best at the 4th epoch, and it’s not very different from the first couple epochs. This means the model does not improved a lot after initiation. Does this mean the data set is just very hard to be trained?

Screen Shot 2021-03-05 at 10.14.55 AM2120×660 121 KB
model
class Net(torch.nn.Module):

    def __init__(self, n_class):
    
        super(Net, self).__init__()
        
        self.n_class = n_class
        
        self.Conv1 = nn.Conv1d(in_channels=4,
                               out_channels=320,
                               kernel_size=30)
                               
        self.Conv2 = nn.Conv1d(in_channels=320,
                               out_channels=160,
                               kernel_size=12)
                               
        self.Maxpool = nn.MaxPool1d(kernel_size=13,
                                    stride=11)
                                    
        self.Drop1 = nn.Dropout(0.1)
        
        self.BiLSTM = nn.LSTM(input_size=87, hidden_size=40,
                              num_layers=2,
                              batch_first=True,
                              dropout=0.4,
                              bidirectional=True)
                              
        self.Linear1 = nn.Linear(12800, 256)
        
        self.Linear2 = nn.Linear(256, 256)
        
        self.Linear3 = nn.Linear(256, self.n_class)

        self.Drop2 = nn.Dropout(0.2)

    def forward(self, input):
        x = self.Conv1(input)
        x = F.relu(x)
        x = self.Conv2(x)
        x = F.relu(x)

        x = self.Maxpool(x)

        x = self.Drop2(x)

        x, _ = self.BiLSTM(x)
        #print(f'output shape from LSTM layer: {x.shape}')
        # x = self.attention(x,x,x)

        x = torch.flatten(x, 1)
        #print(f'output shape from flatten layer: {x.shape}')
        x = self.Linear1(x)

        x = F.relu(x)

        x = self.Linear2(x)
        x = F.relu(x)
        x = self.Linear3(x)
        x = torch.sigmoid(x)
        return x

    def __str__(self):
        """
        Just like any class in Python, you can also define custom method on PyTorch modules
        """
        summary(Complex_DanQ, (4, 1000))

Code to train the model
def train(self):
    """ train """

    device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
    model = self._model.to(device)

    train_loss_hist = []
    eval_loss_hist = []

    train_acc_hist = []
    eval_acc_hist = []

    with open(os.path.join(self.plot_path, "train_vali_record.txt"), 'w') as file:
        file.write("Epoch \t Data \t Loss \t Acc \n")
        file.close()

    best_model_wts = copy.deepcopy(model.state_dict())
    best_acc = 0.0

    if self.weight == "NA":
        criterion = nn.BCELoss().to(device)
    else:
        criterion = nn.BCELoss(weight = torch.tensor([self.weight])).to(device) ## for cluster 1(none dominant)

    optimizer = torch.optim.Adam(model.parameters(), lr=self._learning_rate, weight_decay=self._weight_decay)
    train_loader = torch.utils.data.DataLoader(self.train_data, batch_size=self._batch_size, shuffle=True)
    eval_loader = torch.utils.data.DataLoader(self.eval_data, batch_size=self._batch_size, shuffle=True)

    #model.apply(self._weights_init_uniform_rule) # TODO Uniform Initialization added on 0303

    for epoch in range(self._num_epochs):
        print('Epoch {}/{}'.format(epoch, self._num_epochs - 1))
        print('-' * 10)

        # Train

        train_loss = 0.0
        train_acc = 0.0
        model.train()  # set to train mode, use dropout and batchnorm ##

        for X, y in tqdm(train_loader):

            X = X.to(device)
            y = y.to(device)

            # Forward pass: Compute predicted y by passing x to the model
            y_pred_prob = model(X.float()) # predicted value
            #_, preds = torch.max(y_pred, 1) # TODO understand this line

            # Compute and print loss
            loss = criterion(y_pred_prob.float(), y.float())

            # Backward and optimize
            # Zero gradients, perform a backward pass, and update the weights.
            optimizer.zero_grad() # clears old gradients from the last step
            loss.backward() # for each parameter, calculate d(loss)/d(weight)
            optimizer.step() # update weights, causes the optimizer to take a step based on the gradients of the parameters

            # statistics
            y_pred = (y_pred_prob > 0.5).float()  ## 0/1
            train_loss += loss.item() * X.size(0) #loss.item() has be reducted by batch size

            train_acc += torch.sum(y_pred == y)

        if epoch % 1 == 0:
            train_loss = train_loss/len(train_loader.dataset)
            train_acc = train_acc.double() / len(train_loader.dataset)
            print('Epoch {} {} Loss: {:.4f} Acc: {:.4f}'.format(epoch, 'train', train_loss, train_acc))

            with open(os.path.join(self.plot_path, "train_vali_record.txt"), 'a') as file:
                file.write("{} \t {} \t {:.4f} \t {:.4f} \n".format(epoch, 'train', train_loss, train_acc))
                file.close()



        train_loss_hist.append(train_loss)
        train_acc_hist.append(train_acc)

        # Evaluation
        eval_loss = 0
        eval_acc = 0

        model.eval() # added to not use drop out and batch norm for validation
        with torch.no_grad(): # disable gradiant calculation
            for X, y in tqdm(eval_loader):
                optimizer.zero_grad() # make sure training and eval has minimum diff
                X, y = X.to(device), y.to(device)
                y_pred_prob = model(X.float()) #TODO check whether the weights updated here since optimizer.step()
                eva_loss = criterion(y_pred_prob.float(), y.float())

                y_pred = (y_pred_prob > 0.5).float() ## 0/1

                # statistics
                eval_loss += eva_loss.item() * X.size(0)
                eval_acc += torch.sum(y_pred == y)

            eval_loss = eval_loss / len(eval_loader.dataset)
            eval_acc = eval_acc.double() / len(eval_loader.dataset)

            print('{} Loss: {:.4f} Acc: {:.4f}'.format("validation", eval_loss, eval_acc))
            with open(os.path.join(self.plot_path, "train_vali_record.txt"), 'a') as file:
                file.write("{} \t {} \t{:.4f} \t {:.4f} \n".format(epoch, 'validation', eval_loss, eval_acc))
                file.close()

        eval_loss_hist.append(eval_loss)
        eval_acc_hist.append(eval_acc)

        if eval_acc > best_acc:
            best_acc = eval_acc
            best_model_wts = copy.deepcopy(model.state_dict())


    # load best model weights
    model.load_state_dict(best_model_wts)

    torch.save(model.state_dict(), self._model_path)
    self._plot_metrics(train_loss_hist, eval_loss_hist, "loss_history.pdf")
    self._plot_metrics(train_acc_hist, eval_acc_hist, "acc_history.pdf")