Hi everybody,
I have a similar problem. I have trained my model (a CNN with batch normalizations and dropouts) and stopped the learning process after I reached ~80% on the validation set (using dataloader with suffle=True). I saved my model with
torch.save(self.model.state_dict(), filename)

I wrote another script, with the following code:
def simple_validator(device, model_file, data_folder, chunk_size):
    val_data = riq.IQDataset(data_folder=data_folder, chunk_size=chunk_size, validation=True)
    val_data.normalize(torch.tensor([-3.1851e-06, -7.1862e-07]), torch.tensor([0.0002, 0.0002]))
            
    model = brain.CharmBrain(chunk_size)
    model.load_state_dict(torch.load(model_file))
    model.to(device)
    model.eval()
            
    tot = 0 
    correct = 0
    with torch.no_grad():
        for chunk, label in val_data: 
            chunk = chunk.to(device, non_blocking=True)
            output = model(chunk.unsqueeze(0))
            _, predicted = torch.max(output, dim=1) 
            if predicted.item() == label:
                correct += 1
            tot += 1
    print(f"Accuracy: {correct/tot}")
                    
                        
def validator(device, model_file, data_folder, chunk_size):
    val_data = riq.IQDataset(data_folder=data_folder, chunk_size=chunk_size, validation=True)
    val_data.normalize(torch.tensor([-3.1851e-06, -7.1862e-07]), torch.tensor([0.0002, 0.0002]))
    val_loader = torch.utils.data.DataLoader(val_data, batch_size=512, shuffle=True, num_workers=20, pin_memory=True)
                
    model = brain.CharmBrain(chunk_size) 
    model.load_state_dict(torch.load(model_file))
    model.to(device)
    model.eval()
    
    tot = 0
    correct = 0
    with torch.no_grad():
        for chunks, labels in val_loader:
            chunks = chunks.to(device, non_blocking=True)
            labels = labels.to(device, non_blocking=True)
            output = model(chunks)
            _, predicted = torch.max(output, dim=1)
            correct += int((predicted == labels).sum())
            tot += labels.shape[0]
        print(f"Accuracy: {correct/tot}")
     

The two functions (validator and simple_validator) operate with the same model parameter file and the same data; however:

multiple runs of simple_validator return the same value,  ~0.336
multiple runs of validator return different values, but all around ~0.81
multiple runs of validator setting shuffle=False return different values, but all around 0.3365

(Note, this model is a classifier with 3 classes, so accuracy of 0.33 means random output)
Any idea, where the bug could be?
Thanks,
Luca