I think you can define a new class , and define the vgg model and an extension linear layer separately inside that class. Then, in the forward() function, you get features from both sources (vgg features and hand-crafted features), and concatenate them and send them to the classifier. This way, the backpropagation will work on both parts.
def handcrafted_features(images):
    # Your hand-crafted features here (I am just using random numbers for now)
    hand_feat = np.random.normal(size=(len(images), 100))
    hand_feat = np.array(hand_feat, dtype='float32')
    return torch.tensor(hand_feat)

class VGGXtend(nn.Module):
    def __init__(self, num_hand_features, num_classes):
        super(VGGXtend, self).__init__()

        # define the vgg model:
        self.vgg = models.vgg16(pretrained=True)
        # change the last FC layer:
        self.vgg.classifier[6] = nn.Linear(4096, 1000, bias=True)

        # define the new classifier:
        self.classifier = nn.Linear(1000 + num_hand_features, num_classes)

    def forward(self, images):
        vgg_feat = self.vgg(images)
        hand_feat = handcrafted_features(images)
        features = torch.cat([vgg_feat, hand_feat], dim=1)
        print(features.shape)
        return self.classifier(features)

and a test example:
vggx = VGGXtend(100, 100)
images = np.random.normal(size=(16, 3, 224, 224))
images = np.array(images, dtype='float32')
vggx(torch.tensor(images))