I’m fairly new to PyTorch and am playing around with training on a GPU for the first time.
I’m using a pre-trained Densenet121. Unfrozen layers:
Sequential(
  (fc1): Linear(in_features=1024, out_features=500, bias=True)
  (relu): ReLU()
  (fc2): Linear(in_features=500, out_features=81, bias=True)
  (output): Softmax()
)

Anyway, training is working fine (though still fairly slow considering) but when I starting calculating the Validation Loss and Accuracy, the training slows down dramatically. Maybe 3-4X slower. Is there anything I am doing wrong to cause this? What might I do to improve training speed with validation?
Code below:
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.classifier.parameters(), lr = 0.001)

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
torch.cuda.is_available()

# in-training validation
def validation(model, testloader, criterion):
    test_loss = 0
    accuracy = 0
    for inputs, labels in testloader:
        inputs, labels = inputs.to(device), labels.to(device)

        outputs = model.forward(inputs)
        test_loss += criterion(outputs, labels).item()

        ps = torch.exp(outputs)
        equality = (labels.data == ps.max(dim=1)[1])
        accuracy += equality.type(torch.cuda.FloatTensor).mean()
    
    return test_loss, accuracy

model.to(device)

for epoch in range(epochs):
    running_loss = 0
    model.train()

    for ii, (inputs, labels) in enumerate(trainloader):
        steps += 1
        optimizer.zero_grad()
        inputs, labels = inputs.to(device), labels.to(device)
        outputs = model.forward(inputs)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()
        running_loss += loss.item()
        
        if steps % print_steps == 0:
            model.eval()
            with torch.no_grad():
                test_loss, accuracy = validation(model, testloader, criterion)
            
            print("Epoch: {}/{}.. ".format(epoch+1, epochs),
                  "Training Loss: {:.3f}.. ".format(running_loss / print_steps),
                  "Test Loss: {:.3f}.. ".format(test_loss / len(testloader)),
                  "Test Accuracy: {:.3f}".format(accuracy / len(testloader)))
            
            running_loss = 0
            model.train()