Yes, we are talking about the same use case. It does not matter whether I take the training or the validation dataset, the validation loss is always bad whereas the training loss is good.
        for t in range(initial_epoch, epochs):
            
            # TRAINING
            self.train()
            train_epoch_loss = 0.0
            train_epoch_acc = 0
            # Run batches
            for batch_i, batch_data in enumerate(train_data):  
                # Get batch data
                X_train_batch = batch_data[0].float()       
                y_train_batch = batch_data[1].long()        
                # GPU access
                X_train_batch, y_train_batch = X_train_batch.to(device), y_train_batch.to(device)
                # Backprop
                opt.zero_grad()
                y_train_pred = self(X_train_batch)
                train_loss = loss(y_train_pred, y_train_batch)     
                train_loss.backward()
                opt.step()
                # Update status
                train_epoch_loss += train_loss.item()
                for param in self.parameters():
                    param.requires_grad = True
                # acc
                _, train_preds = torch.max(y_train_pred, dim=1)
                train_epoch_acc += torch.sum(train_preds == y_train_batch)

            # VALIDATION
            val_epoch_loss = 0.0
            val_epoch_acc = 0
            if X_val is not None and y_val is not None:
                val_data = get_loader(X_val, y_val, batch_size, shuffle)
                self.eval()
                with torch.no_grad():

                    for batch_i, batch_data in enumerate(val_data):
                        # Get batch data
                        X_val_batch = batch_data[0].float()  
                        y_val_batch = batch_data[1].long()  
                        # GPU access
                        X_val_batch, y_val_batch = X_val_batch.to(device), y_val_batch.to(device)
                        y_val_pred = self(X_val_batch)
                        # loss
                        val_loss = loss(y_val_pred, y_val_batch)
                        val_epoch_loss += val_loss.item()
                        # acc
                        _, val_preds = torch.max(y_val_pred, dim=1)
                        val_epoch_acc += torch.sum(val_preds == y_val_batch)

This is the part of my fit method with training and validation.