I am refusing to admit defeat as I really want to use PyTorch for this project, I have tested this very same model using Tensorflow on  4 different machines and I get exactly the same results, always performs well with accuracy of around 93 - 98% and on unseen data it generally only gets 1 -3 false classifications.
Could you see if there is anything wrong with my code please as no matter what I try I am always getting this issues.
import torch.nn as nn
import torch.nn.functional as F

class Net(nn.Module):
    def __init__(self):
        super().__init__()
        
        self.pad1 = nn.ZeroPad2d(padding=2)
        self.conv1 = nn.Conv2d(3, 30, kernel_size=5, stride=1)
        
        self.pad2 = nn.ZeroPad2d(padding=2)
        self.conv2 = nn.Conv2d(30, 30, kernel_size=5, stride=1)
        
        self.pool3 = nn.MaxPool2d(kernel_size=2, stride=2, padding=0)
        
        self.flatten = nn.Flatten()
        
        self.fc1 = nn.Linear(75000, 128)
        
        self.relu = nn.ReLU()
        self.softmax = nn.Softmax()

    def forward(self, x):
        x = self.pad1(x)
        x = self.relu(self.conv1(x))
        
        x = self.pad2(x)
        x = self.relu(self.conv2(x))
        
        x = self.pool3(x)
        x = self.flatten(x)
        x = self.fc1(x)
        
        x = self.softmax(x)
        return x


net = Net()
print(net)

train_size = int(split * len(dataset))
test_size = len(dataset) - train_size

train_set, validation_set = torch.utils.data.random_split(dataset,[train_size, test_size])

print(len(train_set))
print(len(validation_set))

train_loader = DataLoader(dataset=train_set, shuffle=shuffle, batch_size=batch_size,
                          num_workers=num_workers, pin_memory=pin_memory)

validation_loader = DataLoader(dataset=validation_set, shuffle=shuffle, batch_size=batch_size,
                               num_workers=num_workers, pin_memory=pin_memory)

import torch.optim as optim

loss_fn = nn.CrossEntropyLoss()
optimizer = optim.Adam(net.parameters(), lr=learning_rate, weight_decay=decay)

train_accu = []
train_losses = []

for epoch in range(epochs): 

    net.train()
    
    running_loss=0
    correct=0
    total=0
    
    for i, data in enumerate(train_loader, 0):
        # get the inputs; data is a list of [inputs, labels]
        inputs, labels = data

        # zero the parameter gradients
        optimizer.zero_grad()

        # forward + backward + optimize
        outputs = net(inputs)
        loss = loss_fn(outputs, labels.long())
        loss.backward()
        optimizer.step()
 
        running_loss += loss.item()

        _, predicted = outputs.max(1)
        total += labels.size(0)
        correct += predicted.eq(labels).sum().item()
       
    train_loss=running_loss/len(train_loader)
    accu=100.*correct/total

    train_accu.append(accu)
    train_losses.append(train_loss)
    print('Train Loss: %.3f | Accuracy: %.3f'%(train_loss,accu))

print('Finished Training')

Results:
Train Loss: 4.847 | Accuracy: 26.551
Train Loss: 4.819 | Accuracy: 62.779
Train Loss: 4.692 | Accuracy: 70.471
Train Loss: 4.393 | Accuracy: 70.471
Train Loss: 4.197 | Accuracy: 70.471
Train Loss: 4.173 | Accuracy: 70.471
Train Loss: 4.161 | Accuracy: 70.471
Train Loss: 4.157 | Accuracy: 70.471
Train Loss: 4.159 | Accuracy: 70.471
Train Loss: 4.158 | Accuracy: 70.471
Train Loss: 4.156 | Accuracy: 70.471
Train Loss: 4.156 | Accuracy: 70.471
Train Loss: 4.164 | Accuracy: 70.471
Train Loss: 4.164 | Accuracy: 70.471
Train Loss: 4.157 | Accuracy: 70.471
Train Loss: 4.163 | Accuracy: 70.471
Train Loss: 4.157 | Accuracy: 70.471
Train Loss: 4.157 | Accuracy: 70.471
Train Loss: 4.157 | Accuracy: 70.471
Train Loss: 4.167 | Accuracy: 70.471
Train Loss: 4.157 | Accuracy: 70.471
Train Loss: 4.157 | Accuracy: 70.471
Train Loss: 4.157 | Accuracy: 70.471
Train Loss: 4.160 | Accuracy: 70.471
Train Loss: 4.160 | Accuracy: 70.471
Train Loss: 4.158 | Accuracy: 70.471
Train Loss: 4.160 | Accuracy: 70.471
Train Loss: 4.163 | Accuracy: 70.471
Train Loss: 4.158 | Accuracy: 70.471
Train Loss: 4.160 | Accuracy: 70.471
Train Loss: 4.161 | Accuracy: 70.471
Train Loss: 4.161 | Accuracy: 70.471
Train Loss: 4.165 | Accuracy: 70.471
Train Loss: 4.165 | Accuracy: 70.471
Train Loss: 4.170 | Accuracy: 70.471
Train Loss: 4.156 | Accuracy: 70.471
Train Loss: 4.156 | Accuracy: 70.471

class AllIdbDataset(Dataset):
    """
    Pytorch dataset class

    Args:
        csv_path (string): path to CSV
        img_path (string): path to the training folder
        transform: Pytorch transforms
    """
    
    def __init__(self, data_dir, csv_path, transform=None):
        
        self.data_dir = data_dir
        self.data = pd.read_csv(csv_path)
        self.transform = transform

    def __len__(self):
        return len(self.data)

    def __getitem__(self, index):
        img_id = self.data.iloc[index, 0]
        img = Image.open(os.path.join(self.data_dir, img_id)).convert("RGB")
        label = torch.tensor(float(self.data.iloc[index, 1]))

        if self.transform is not None:
            img = self.transform(img)

        return (img, label)

It must be something noob I am doing, there is no way that this classifier is doing this bad without a mistake I made myself.
I am not using transforms except for converting to tensor, but using custom data augmentation that again has been used in my projects for years for with no issues.
Maybe you can see something I am missing.