I realized that when a backbone is built with BackboneWithFPN(), I would need to make anchor_generator and specify the sizes and aspect_ratio with tuples (single-level, not nested tuples). However, when not using BackboneWithFPN(), the anchor_genertor needs a nested tuple; i.e. Tuple[Tuple[int]]. I have made the following script to play around mobilenet backbone with or without FPN:
"""
MobileNet Backbone for MaskRCNN
script: backbones.py
  - python -m backbones --use_fpn=0
  - python -m backbones --use_fpn=1
"""

import sys
import argparse

import torch
from torch import nn
import torchvision
from torchvision.ops import misc as misc_nn_ops
from torchvision.ops.feature_pyramid_network import LastLevelMaxPool
from torchvision.models import mobilenet
from torchvision.models.detection.rpn import AnchorGenerator
from torchvision.models.detection.backbone_utils import BackboneWithFPN
from torchvision.models.detection.mask_rcnn import MaskRCNN

from ops import model_ops

def get_mobilenet_backbone(
        use_fpn: bool = True):

    # create the backbone
    bb = mobilenet.__dict__['mobilenet_v3_small'](
        pretrained=True,
        norm_layer=misc_nn_ops.FrozenBatchNorm2d)
    backbone = bb.features

    # backbone = torchvision.models.mobilenet_v2(pretrained=True).features
    backbone.out_channels = 1280

    stage_indices = [0] + [
        i for i, b in enumerate(backbone)
        if getattr(b, "_is_cn", False)] + [len(backbone) - 1]
    print(stage_indices)
    num_stages = len(stage_indices)
    trainable_layers = 2

    if trainable_layers == 0:
        freeze_before = num_stages
    else:
        freeze_before = stage_indices[num_stages - trainable_layers]

    for b in backbone[:freeze_before]:
        for parameter in b.parameters():
            parameter.requires_grad_(False)

    out_channels = 1280
    extra_blocks = LastLevelMaxPool()


    if use_fpn:
        returned_layers = [num_stages - 2, num_stages - 1]
        assert min(returned_layers) >= 0 and max(returned_layers) < num_stages
        return_layers = {f'{stage_indices[k]}': str(v) for v, k in enumerate(returned_layers)}

        in_channels_list = [backbone[stage_indices[i]].out_channels for i in returned_layers]
        backbone = BackboneWithFPN(
            backbone, return_layers,
            in_channels_list, out_channels,
            extra_blocks=extra_blocks)
    else:
        backbone = nn.Sequential(
           backbone,
           # depthwise linear combination of channels to reduce their size
           nn.Conv2d(backbone[-1].out_channels, out_channels, 1))
        backbone.out_channels = out_channels

    return backbone


def main(args):
    backbone = get_mobilenet_backbone(args.use_fpn)
    x = torch.randn(3, 64, 64)
    out = backbone(x.unsqueeze(0))
    if args.use_fpn:
        print('Output of backbone with FPN:', out.keys(), [out[k].shape for k in out])
    else:
        print('Output of backbone without FPN:', out.shape)

    if args.use_fpn:
        anchor_generator = AnchorGenerator(
            sizes=(32, 64, 128),
            aspect_ratios=(0.5, 1.0, 2.0))
    else:
        anchor_generator = AnchorGenerator(
            sizes=((32, 64, 128),),
            aspect_ratios=((0.5, 1.0, 2.0),))

    box_roi_pooler = torchvision.ops.MultiScaleRoIAlign(
        featmap_names=['0'], output_size=7, sampling_ratio=2)

    mask_roi_pooler = torchvision.ops.MultiScaleRoIAlign(
        featmap_names=['0'], output_size=14, sampling_ratio=2)

    model = MaskRCNN(
        backbone, num_classes=2,
        rpn_anchor_generator=anchor_generator,
        box_roi_pool=box_roi_pooler,
        mask_roi_pool=mask_roi_pooler)

    print('model:', model)
    model.eval()
    out = model([x])
    key = 'masks'
    print('Output of model:', [f'\'masks\':{v[key].shape}' for v in out])


def parse(argv):
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--use_fpn', type=int, required=False,
        choices=[0, 1])

    args = parser.parse_args()
    args.use_fpn = bool(args.use_fpn)

    return args


if __name__ == '__main__':
    args = parse(sys.argv[1:])
    main(args)