I did the following, but this is kinda stupid, but it apparently works.
Please let me know if there is any other solutions
import torch
import torch.nn as nn
from vgg import VGG16 as vgg


InputChannel = 5

bases = {
    'A': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'C', 512, 512, 512, 'M', 512, 512, 512]}
base = vgg(bases['A'], InputChannel)

vgg = nn.ModuleList(base)
#reading pretrained weights
vgg_weights = torch.load('weights/vgg16_reducedfc.pth')
# Changing the pretrain weights to have the same 5 channels
Main = vgg_weights['0.weight']
SIZEMain = Main.size()
Zeros = torch.zeros(SIZEMain[0],2,SIZEMain[2],SIZEMain[3]).cpu()
NewMain = torch.cat((Main,Zeros),1) 
#Update the corresponding weight in state
state = vgg.state_dict()
state.update(vgg_weights)
state['0.weight'] = NewMain
#Give State to VGG
vgg.load_state_dict(state)