You’re correct! I do need a warmup for my gpu. The processing time per image has decreased to 3-4 ms.
For the overfitting, I have tried dropout(drop rate 0.5) but it did work. Here is my code for build the last classifier:
#input size is 224x224
model = models.resnet152(pretrained=True)
num_ftrs = model.fc.in_features
model.fc = build_classifier(num_ftrs, [1050,500], 196)

def build_classifier(num_in_features, hidden_layers, num_out_features):
    classifier = nn.Sequential()
    # when we don't have any hidden layers
    if hidden_layers == None:      
        classifier.add_module('fc0', nn.Linear(num_in_features, 196))    
    #when we have hidden layers
    else:      
        layer_sizes = zip(hidden_layers[:-1], hidden_layers[1:])
        classifier.add_module('fc0', nn.Linear(num_in_features, hidden_layers[0]))
        classifier.add_module('relu0', nn.ReLU())
        classifier.add_module('drop0', nn.Dropout(.5))
        
        for i, (h1, h2) in enumerate(layer_sizes):
            classifier.add_module('fc'+str(i+1), nn.Linear(h1, h2))
            classifier.add_module('relu'+str(i+1), nn.ReLU())
            classifier.add_module('drop'+str(i+1), nn.Dropout(.5))
        
        classifier.add_module('output', nn.Linear(hidden_layers[-1], num_out_features))
        
    return classifier

Did I do something wrong?