Hello, I am new to mmdetection and model optimization. I am using mmdetection to use mask RCNN with SWIN transformer as the backbone. As the model is big I was trying to reduce its size.As far as I know , pruning as implemented in pytorch would not change the model size but would simply create a mask.I have also tried L1 unstructured quantization with success and was currently trying low rank transformations. I can see the transformation happening when I compare the print(model) before and after the low-rank transformation. However, the model is not able to infer anything. I see no bounding box that appear. The following is what I have tried :
config='/content/mmdetection/configs/swin/mask_rcnn_swin-t-p4-w7_fpn_1x_coco.py'
# Setup a checkpoint file to load
checkpoint = 'checkpoints/mask_rcnn_swin-t-p4-w7_fpn_1x_coco_20210902_120937-9d6b7cfa.pth'
# initialize the detector
model = init_detector(config,checkpoint)

Once the model is initialized I use the following for low-rank transformation
def low_rank_transform(model):
    svd_classifier_layers=[]
    L=40
    for module_name, module in model.named_children():
      if module_name=='proj':
        continue
      if len(list(module.children())) > 0:
        replace_layers(module)
      if isinstance(module, torch.nn.Linear):
        W=module.weight.data
        # print(W)
        # U,S,V=torch.svd(W)
        U, S, Vh = torch.linalg.svd(W)
        V = Vh.transpose(-2, -1).conj()
        W1 = U[:,:L]
        W2 = torch.diag(S[:L]) @ V[:,:L].t()
        if module.bias is not None:
          # print("W1:",W1)
          layer_1 = nn.Linear(in_features=module.in_features,
                            out_features=L, bias=True)
          layer_1.weight.data = W2
          svd_classifier_layers.append(layer_1)

          layer_2 = nn.Linear(in_features=L,
                            out_features=module.out_features , bias=True)
          layer_2.weight.data = W1
          layer_2.bias.data = module.bias.data
          svd_classifier_layers.append(layer_2)
          # print("Layer2",layer_2.weight.data)
          # break

        else:
          layer_1 = nn.Linear(in_features=module.in_features,
                            out_features=L, bias=False)
          layer_1.weight.data = W2
          svd_classifier_layers.append(layer_1)

          layer_2 = nn.Linear(in_features=L,
                            out_features=module.out_features , bias=False)
          layer_2.weight.data = W1
          # layer_2.bias.data = module.bias.data
          svd_classifier_layers.append(layer_2)

        try:
          # print("we are now trying to replace",module,"with",svd_classifier_layers,"in a sequnetial way")
          # n =int(module_name)
          # model[n]=nn.Sequential(*svd_classifier_layers)
          setattr(model, module_name, nn.Sequential(*svd_classifier_layers))
        except:
          setattr(model, module_name, nn.Sequential(*svd_classifier_layers))

Then I simply call the above function by passing only the backbone of the model.
low_rank_transform(model):(model.backbone)
I can see the expected change in the architecture where a single Linear layer gets replaced by two linear layers. However, nothing happens during inference.
Can someone help me with what is that I might be doing wrong here?