KFrank:

the behaviors of the loss and the accuracy aren’t consistent with one another


Your right, I didn’t make the connection between test loss and accuracy. A quick google search results in some posts about similar issues, but offers no expiations as to how to solve it or what is causing it, only inferring that the model is overfitting.



 KFrank:

My complaint was that your test accuracy became significantly
better than your training accuracy (in your original graphs, around
epoch 30)


The accuracy increased by around 20%, only for a few batches. I can only infer this is because the validation set only has 33 samples.



 KFrank:

Review your code carefully, line by line. Is the calculation you are
doing one that makes sense? Are you actually doing the calculation
you think you are doing, or has a typo crept in somewhere?


I double checked each line in my training class, couldn’t find anything. I trained on the MNIST dataset with my training class and the results look perfect.
download786×595 44 KB



 KFrank:

But: Do everything else in your calculation  exactly  the same way.
Don’t change your code to make a quick hack to get the “test results”
from your training data – process your 33 “fake” samples exactly the
same way you process your real ones – any kind of preprocessing,
etc. Pass them through the exact same test-result evaluation code,
and so on.


I changed the the test set to 33 samples of the training set, and these are the results:
Traning2000×1428 118 KB
Thank you for helping!
Training Class code:
class Train():
    def __init__(self, model, datasets, epochs, batch_size, criterion, optimizer, tensorboard=True, device_type='cuda', metrics_path="/content/drive/Shared drives/EEG_Aditya/model-results/"):
        torch.manual_seed(1)
        self.model = model
        self.epochs = epochs
        self.criterion = criterion
        self.optimizer = optimizer
        self.metrics_path = metrics_path
        self.shape = np.array([np.array(i) for i in datasets[0][1][0]]).shape
        self.model_name = f"{model.__class__.__name__}--{str(datetime.datetime.now()).replace(' ', '--').replace(':', '-').split('.')[0]}"
        print(self.model_name)
        time.sleep(1)
        self.device(device_type)
        self.data_loader(datasets, batch_size)
        
        if tensorboard:
            self.tb = SummaryWriter()

        #self.train()
        test_acc, test_loss = self.test(self.test_dataloader, log=True)
        print(f"Test Accuracy: {round(test_acc, 4)}, Test Loss: {round(test_loss,4)}")

    def device(self, device):
        self.device_type = device
        if device == "cuda":
            self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
            self.model.to(self.device)

    def save_model(self):
        torch.save(self.model.state_dict(), f"{self.metrics_path}parameters/{self.model_name}.pt")

    def logger(self, epoch, batch, train_loss, train_acc, val_loss, val_acc):
        if os.path.exists(f"{self.metrics_path}metrics/{self.model_name}.log") == False:
            with open(f"{self.metrics_path}metrics/{self.model_name}.log", 'a') as file:
                file.write(str(self.shape)+"\n")
                file.write(str(self.model)+"\n")
                file.write(str(self.criterion)+"\n")
                file.write(str(self.optimizer)+"\n")
                file.write("epochs,batch,training loss,training accuracy,validation loss,validation accuracy\n")

        with open(f"{self.metrics_path}metrics/{self.model_name}.log", 'a') as file:
            file.write(f"{epoch},{batch},{round(float(train_loss),4)},{round(float(train_acc),4)},{round(float(val_loss),4)},{round(float(val_acc),4)}\n")

    def data_loader(self, datasets, batch_size):
        if self.device_type=="cuda":
            self.train_dataloader = torch.utils.data.DataLoader(datasets[0], batch_size, shuffle=True)
            self.val_dataloader = torch.utils.data.DataLoader(datasets[1], batch_size)
            self.test_dataloader = torch.utils.data.DataLoader(datasets[2], batch_size)

    def train(self):
        min_loss = self.test(self.test_dataloader, log=False)[1]
        for epoch in tqdm(range(self.epochs)):
            for batch_i,(data, targets) in enumerate(self.train_dataloader):
                # Training
                data = data.to(self.device)
                targets = targets.to(self.device)

                train_loss, train_acc = self.forward_pass(data, targets, train=True)

                # Validation
                val_accs, val_losses = 0,0
                for val_batch_i, (val_data, val_targets) in enumerate(self.val_dataloader):
                    val_data = val_data.to(self.device)
                    val_targets = val_targets.to(self.device)
                    val_loss, val_acc = self.forward_pass(val_data, val_targets, train=False)
                    val_accs += val_acc
                    val_losses += val_loss
                val_accs, val_losses = val_accs/(val_batch_i+1), val_losses/(val_batch_i+1)

                # Saving model with lowest validation loss
                if min_loss >= val_losses:
                    self.save_model()
                    min_loss = val_losses

                # Logs model metrics
                self.logger(epoch, batch_i, train_loss, train_acc, val_losses, val_accs)

            # Tensorboard
            self.tensorboard([['Training Loss', train_loss, epoch],
                              ['Validation Loss', val_losses, epoch],
                              ['Traning Accuracy', train_acc, epoch],
                              ['Validation Accuracy', val_accs, epoch],
                              ['Learning Rate', self.optimizer.param_groups[0]['lr'], epoch]])
        
        print("Finished Training")
        print(f"Training Accuracy: {round(train_acc, 4)}, Training Loss: {round(train_loss,4)}")
        print(f"Validation Accuracy: {round(val_accs, 4)}, Validation Loss: {round(val_losses, 4)}")


    def forward_pass(self, data, targets, train=False):

        if train:
            self.model.zero_grad()
            self.model.train()
        else:
            self.model.eval()

        with (torch.enable_grad() if train else torch.no_grad()):
            outputs = self.model(data)
            matches = [torch.argmax(i)==(j) for i,j in zip(outputs, targets)]
            loss = self.criterion(outputs, targets.long())
            acc = matches.count(True)/len(matches)

        if train:
            loss.backward()
            self.optimizer.step()

        del outputs
        del matches
        return loss.item(), acc

    def test(self, dataloader, log=True):
        accs, losses = 0,0
        for batch_i, (data, targets) in enumerate(dataloader):
            data = data.to(self.device)
            targets = targets.to(self.device)
            loss, acc = self.forward_pass(data, targets.long(), train=False)
            accs += acc
            losses += loss
        accs, losses = accs/(batch_i+1), losses/(batch_i+1)
        if log:
            self.logger(-1, -1, -1, -1, losses, accs)
        return accs, losses

    def tensorboard(self, *scalars):
        for name, scalar, epoch in scalars[0]:
            self.tb.add_scalar(name, scalar, epoch)