So I modifies my code a bit to the following. Still it doesn’t seem to work
    pretrained = models.vgg16(pretrained=True)
   
    vgg = my_model()# vgg16 with 4th pool removed
    vgg_dict = vgg.state_dict()

    for k, v in pretrained_dict.items():
        if int(k.split('.')[1]) > 22:
            break

        vgg_dict[k] = v
     

    vgg_dict['features.23.weight'] = pretrained_dict['features.24.weight']
    vgg_dict['features.23.bias'] = pretrained_dict['features.24.bias']
    vgg_dict['features.25.weight'] = pretrained_dict['features.26.weight']
    vgg_dict['features.25.bias'] = pretrained_dict['features.26.bias']
    vgg_dict['features.27.weight'] = pretrained_dict['features.28.weight']
    vgg_dict['features.27.bias'] = pretrained_dict['features.28.bias']