Hello, abenju!
I can suggest you to have a look for mask rcnn finetuning tutorial.
https://pytorch.org/tutorials/intermediate/torchvision_tutorial.html 6
And if you want to change inner layers you probably should rewrite RCNN class in torchvision.
  
      github.com
  
  
    pytorch/vision/blob/master/torchvision/models/detection/faster_rcnn.py 4
import torch
from torch import nn
import torch.nn.functional as F

from torchvision.ops import MultiScaleRoIAlign

from ._utils import overwrite_eps
from ..utils import load_state_dict_from_url

from .anchor_utils import AnchorGenerator
from .generalized_rcnn import GeneralizedRCNN
from .rpn import RPNHead, RegionProposalNetwork
from .roi_heads import RoIHeads
from .transform import GeneralizedRCNNTransform
from .backbone_utils import resnet_fpn_backbone, _validate_trainable_layers, mobilenet_backbone


__all__ = [
    "FasterRCNN", "fasterrcnn_resnet50_fpn", "fasterrcnn_mobilenet_v3_large_320_fpn",
    "fasterrcnn_mobilenet_v3_large_fpn"


  This file has been truncated. show original

  
  
    
    
  
  


About making flexible input image sizes, no need to worry, because there already implemented tranformation layers at the beggining and at the and of faster rcnn model. All you need to do is just set min_size and max_size.
Again, have a look at this link:
  
      github.com
  
  
    pytorch/vision/blob/master/torchvision/models/detection/faster_rcnn.py 4
import torch
from torch import nn
import torch.nn.functional as F

from torchvision.ops import MultiScaleRoIAlign

from ._utils import overwrite_eps
from ..utils import load_state_dict_from_url

from .anchor_utils import AnchorGenerator
from .generalized_rcnn import GeneralizedRCNN
from .rpn import RPNHead, RegionProposalNetwork
from .roi_heads import RoIHeads
from .transform import GeneralizedRCNNTransform
from .backbone_utils import resnet_fpn_backbone, _validate_trainable_layers, mobilenet_backbone


__all__ = [
    "FasterRCNN", "fasterrcnn_resnet50_fpn", "fasterrcnn_mobilenet_v3_large_320_fpn",
    "fasterrcnn_mobilenet_v3_large_fpn"


  This file has been truncated. show original

  
  
    
    
  
  


Sorry for my english, hope I could help you somehow.