Does anyone tried to implement FCN using the pretrained vgg16 in model zoo?
I tried and tested on SiftFlow dataset but it can’t achieve the same level of performance of original caffe version or my theano version. Below is the detailed codes how I implemented it. Any suggestions will be welcome! 
In summary, I first implemented a FCN-32 network class and loaded/reshaped pre-trained weights in the same way as caffe did. Before feeding data into the network, every image is scaled to [0 1], subtracted by mean [0.485, 0.456, 0.406], and divided by std [0.229, 0.224, 0.225]. I guess this is the same normalization used for training the model zoo vgg16 network. And the network is trained using the exact same parameters as caffe version.
#-------- main part --------
model = FCN_32()

pth_file = 'vgg16-82412952.pth' # download from model zoo
pre_trained_weights = torch.load(pth_file)

layer_names = [layer_name for layer_name in pre_trained_weights]

counter = 0
for p in model.parameters():
    if counter < 26:  # conv1_1 to pool5
        p.data = pre_trained_weights[ layer_names[counter] ]
    elif counter == 26:  # fc6 weight
        p.data = pre_trained_weights[ layer_names[counter] ].view(4096, 512, 7, 7)
    elif counter == 27:  # fc6 bias
        p.data = pre_trained_weights[ layer_names[counter] ]
    elif counter == 28:  # fc7 weight
        p.data = pre_trained_weights[ layer_names[counter] ].view(4096, 4096, 1, 1)
    elif counter == 31:   # upscore layer
        m, k, h, w = 34, 34, 64, 64
        filter = upsample_filt(h)
        filter = torch.from_numpy(filter.astype('float32'))
        p.data = filter.repeat( m, k, 1, 1 )
    counter += 1

def upsample_filt(size):
    """
        Make a 2D bilinear kernel suitable for upsampling of the given (h, w) size.
        """
    factor = (size + 1) // 2
    if size % 2 == 1:
        center = factor - 1
    else:
        center = factor - 0.5
    og = np.ogrid[:size, :size]
    return (1 - abs(og[0] - center) / factor) * \
        (1 - abs(og[1] - center) / factor)



#-------- FCN-32 implementation part --------
import torch
import torch.nn as nn
import math
import torch.utils.model_zoo as model_zoo


def conv3x3(in_planes, out_planes, stride=1, padding=1):
    "3x3 convolution with padding"
    return nn.Conv2d(in_planes, out_planes, kernel_size=(3,3), stride=(stride,stride),
                     padding=(padding,padding))

def conv1x1(in_planes, out_planes, stride=1):
    "1x1 convolution with padding"
    return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride,
                     padding=0)





class FCN_32(nn.Module):
    def __init__(self):
        super(FCN_32, self).__init__()

        # vgg part
        self.conv1_1 = conv3x3(3, 64, stride=1, padding=100)
        self.conv1_2 = conv3x3(64, 64)
        
        self.conv2_1 = conv3x3(64, 128)
        self.conv2_2 = conv3x3(128, 128)
        
        self.conv3_1 = conv3x3(128, 256)
        self.conv3_2 = conv3x3(256, 256)
        self.conv3_3 = conv3x3(256, 256)

        self.conv4_1 = conv3x3(256, 512)
        self.conv4_2 = conv3x3(512, 512)
        self.conv4_3 = conv3x3(512, 512)

        self.conv5_1 = conv3x3(512, 512)
        self.conv5_2 = conv3x3(512, 512)
        self.conv5_3 = conv3x3(512, 512)
        
        self.fc6 = nn.Conv2d(512, 4096, kernel_size=7, stride=1, padding=0)
        self.dropout = nn.Dropout()
        self.fc7 = nn.Conv2d(4096, 4096, kernel_size=1, stride=1, padding=0)
        
        self.score_fr_sem = nn.Conv2d(4096, 34, kernel_size=1, stride=1, padding=0, bias=False)
        
        self.upscore_sem = nn.ConvTranspose2d(34, 34, kernel_size=64, stride=32, padding=0, output_padding=0, bias=False)

        
        self.pool = nn.MaxPool2d(kernel_size=2, stride=2, ceil_mode=True)



        self.relu = nn.ReLU(inplace=True)
        
        self.softmax = nn.Softmax()
    
        self._initialize_weights()

    def forward(self, x):
        # vgg part
        conv1_1 =  self.relu(  self.conv1_1(x) )
        conv1_2 =  self.relu(  self.conv1_2(conv1_1) )
        pool1 = self.pool(conv1_2)
        
        conv2_1 =  self.relu(   self.conv2_1(pool1) )
        conv2_2 =  self.relu(   self.conv2_2(conv2_1) )
        pool2 = self.pool(conv2_2)
        
        conv3_1 =  self.relu(   self.conv3_1(pool2) )
        conv3_2 =  self.relu(   self.conv3_2(conv3_1) )
        conv3_3 =  self.relu(   self.conv3_3(conv3_2) )
        pool3 = self.pool(conv3_3)
        
        conv4_1 =  self.relu(   self.conv4_1(pool3) )
        conv4_2 =  self.relu(   self.conv4_2(conv4_1) )
        conv4_3 =  self.relu(   self.conv4_3(conv4_2) )
        pool4 = self.pool(conv4_3)
        
        conv5_1 =  self.relu(   self.conv5_1(pool4) )
        conv5_2 =  self.relu(   self.conv5_2(conv5_1) )
        conv5_3 =  self.relu(   self.conv5_3(conv5_2) )
        pool5 = self.pool(conv5_3)
        
        fc6 = self.dropout( self.relu(   self.fc6(pool5) ) )
        fc7 = self.dropout( self.relu(   self.fc7(fc6) ) )
        
        score_fr_sem = self.score_fr_sem( fc7 )
        
        upscore_sem = self.upscore_sem(score_fr_sem)
        
        crop = upscore_sem[:, :, 19:19+256, 19:19+256] # batch, 34, 256, 256
        
        crop = crop.transpose( 1, 3 )
        crop = crop.transpose( 1, 2 ) # batch, 256, 256, 34
        
        output = crop.contiguous().view(-1, crop.size(3))
        
        output = self.softmax(output)
        
        
        return output
        
    
        
    def _initialize_weights(self):
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                m.weight.data.normal_(0, math.sqrt(2. / n))
                if m.bias is not None:
                    m.bias.data.zero_()
            elif isinstance(m, nn.BatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()
            elif isinstance(m, nn.Linear):
                n = m.weight.size(1)
                m.weight.data.normal_(0, 0.01)
                m.bias.data.zero_()