Hello @SimonW, @mfayyaz, @ptrblck, @Khalooei. I have prepared two scripts. one of them is used for training and the other for comparing the output. Here is my main file to overfit on a sample based on resnet18 pre-trained weight.
import torch
import torchvision.models as models
import torch.nn as nn
from torch.autograd import Variable as V
import torch.optim as optim


class MyModel(nn.Module):
	"""docstring for MyModel"""
	def __init__(self, resnet18):
		super(MyModel, self).__init__()
		self.features = nn.Sequential(*list(resnet18.children())[:-1])
		self.f1 = nn.Linear(2048, 4)

	def forward(self, input):
		out = self.features(input).detach()
		return self.f1(out.view(out.size(0),-1))
		
def release_weight(model):
    for param in model.parameters():
        param.requires_grad = True
    return model

def freeze_weight(model):
    for param in model.features.parameters():
        param.requires_grad = False
    return model    

resnet18 = models.resnet18(pretrained = True)
model = MyModel(resnet18)
model = release_weight(model)
model = freeze_weight(model)

parameters = filter(lambda p: p.requires_grad, model.parameters())
optimizer = optim.SGD(parameters, lr=1e-3, weight_decay=0.001, momentum=0.9)

inp = V(torch.randn(1,3,256,256), requires_grad = False)
target = V(torch.randn(1,4))

for i in range(2):
	for j in range(100):
		optimizer.zero_grad()
		out = model(inp)
		loss = ((out - target) ** 2).mean()
		loss.backward()
		optimizer.step()
		print(loss.data[0])
	torch.save(model.state_dict(), './simpleModel{}.pth'.format(i))

and then here is the comparing code:
import torch
import torchvision.models as models
import torch.nn as nn
from torch.autograd import Variable as V

class MyModel(nn.Module):
    """docstring for MyModel"""
    def __init__(self, resnet18):
        super(MyModel, self).__init__()
        self.features = nn.Sequential(*list(resnet18.children())[:-1])
        self.f1 = nn.Linear(2048, 4)

    def forward(self, input):
        out1 = self.features(input)
        return out1, self.f1(out1.view(out1.size(0),-1))

ckpt0 = 'simpleModel0.pth'
ckpt1 = 'simpleModel1.pth'

resnet18 = models.resnet18(pretrained = True)
model0 = MyModel(resnet18)
model0.load_state_dict(torch.load(ckpt0), strict=True)
model0.train(False)


resnet18 = models.resnet18(pretrained = True)
model1 = MyModel(resnet18)
# model1.load_state_dict(torch.load(ckpt1))
model1.train(False)

model0_name = []
for name, param in model0.named_parameters():
    model0_name.append(name)

model1_name = []
for name, param in model1.named_parameters():
	model1_name.append(name)

diff = []
for p1, p2 in zip(model0.parameters(), model1.parameters()):
	diff.append((p1.data - p2.data).sum())

for n in zip(model0_name, model1_name, diff):
	print(n[0], n[1], "{0:.8f}".format(n[2]))

model0_running_variance = []
model0_running_mean = []
for module in model0.modules():
    if isinstance(module, nn.modules.BatchNorm2d):
    	model0_running_mean.append(module.running_mean)
    	model0_running_variance.append(module.running_var)

model1_running_variance = []
model1_running_mean = []
for module in model1.modules():
    if isinstance(module, nn.modules.BatchNorm2d):
    	model1_running_mean.append(module.running_mean)
    	model1_running_variance.append(module.running_var)

print("running_mean difference")
for m1, m2 in zip(model0_running_mean, model1_running_mean):
	print("{0:.8f}".format((m1-m2).sum()))

print("running_var difference")
for m1, m2 in zip(model0_running_variance, model1_running_variance):
	print("{0:.8f}".format((m1-m2).sum()))

inp = V(torch.randn(1,3,256,256))
pretrained_head0, out0 = model0(inp)
pretrained_head1, out1 = model1(inp)


diffpretrained_head = (pretrained_head1 - pretrained_head0).data.abs().sum()
diffout = (out1 - out0).data.abs().sum()

print(diffpretrained_head, diffout)

And again, these scripts proved my claim. Could you please check them up on your system?