here is how I have been getting through with custom backbone networks for resnet 101

class ModelResnet101FasterRCNN(FasterRCNN):
    def __init__(self, data_conf, model_conf):
        print("Creating model")
        backbone_nn = torchvision.models.__dict__[model_conf["hyperParameters"]["net"]](pretrained=True)
        # This line above yields the equivalent of ...
        # backbone_nn = torchvision.models.resnet101(pretrained=True)
        # OR
        # backbone_nn = torchvision.models.wide_resnet101_2(pretrained=True)

        modules = list(backbone_nn.children())[:-1]  # delete the last fc layer.
        backbone_nn = nn.Sequential(*modules)
        for param in backbone_nn.parameters():
            param.requires_grad = False

        # FasterRCNN needs to know the number of
        # output channels in a backbone. For resnet101, it's 2048
        backbone_nn.out_channels = 2048