Mabye you can just output two features at the end of the model.
I’m trying to make a keypoint detection with the tutorial https://pytorch.org/tutorials/intermediate/torchvision_tutorial.html,
I changed some code to get keypoint detection like following:

def get_model_instance_segmentation(num_classes=2, num_keypoints = 4, pretrained=True, mask=False):
    from torchvision.models.detection.faster_rcnn import FastRCNNPredictor
    from torchvision.models.detection.mask_rcnn import MaskRCNNPredictor
    from torchvision.models.detection.keypoint_rcnn import KeypointRCNNPredictor

    # load an instance segmentation model pre-trained pre-trained on COCO
    if mask:
        model_template = torchvision.models.detection.maskrcnn_resnet50_fpn
        model = model_template(pretrained=pretrained,
            num_classes=num_classes)
    else:
        model_template = torchvision.models.detection.keypointrcnn_resnet50_fpn
        model = model_template(pretrained=pretrained,
            num_classes=num_classes, num_keypoints=num_keypoints)

    # get number of input features for the classifier
    in_features = model.roi_heads.box_predictor.cls_score.in_features
    print("In Features: ", in_features)
    # replace the pre-trained head with a new one
    model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)

    if mask:
    # now get the number of input features for the mask classifier
        in_features_mask = model.roi_heads.mask_predictor.conv5_mask.in_channels
        hidden_layer = 256
        # and replace the mask predictor with a new one
        model.roi_heads.mask_predictor = MaskRCNNPredictor(in_features_mask,
                                                        hidden_layer,
                                                        num_classes)
    else:
        in_features_keypoint = model.roi_heads.keypoint_predictor.kps_score_lowres.in_channels
        # print("InFeatures: ", in_features_keypoint, model.roi_heads.keypoint_predictor.out_channels)
        model.roi_heads.keypoint_predictor = KeypointRCNNPredictor(in_features_keypoint, num_keypoints)

    return model

you may take a look at the source code of torchvision.models.detection.keypoint_rcnn.