I tried to implement a FCN with PyTorch ,which I have previously implemented in Torch. The BCE losses drops to around 0.05 after several epoches, but when I simply run a inference, even an image in the training dataset, the result is just something mess:

really can not understand why. Any help?
The main training loop:(I use my own image dataloader and training loop, label is either 1 or 0)
optimizer = optim.Adam(fcn.parameters(), lr=opt.lr)


def train(epoch):
	np.random.shuffle(name_list)
	epoch_lossSeg = 0.0
	for iteration in range(0, dataset_size,opt.batchSize):
		batch_img_tensor = torch.FloatTensor(opt.batchSize, 3, opt.h, opt.w)
		batch_annotation_tensor = torch.FloatTensor(opt.batchSize,1,opt.h, opt.w)
		# get a batch input and label
		for  i in range(iteration+0, iteration + opt.batchSize):		
			img_ = Image.open(opt.train_img_path + '/' + name_list[i])
			img = img_.resize(img_size,Image.BILINEAR)
			batch_img_tensor[ i-iteration ] = torch.from_numpy(np.array(img)).float()
			annotation_ = Image.open(opt.train_label_path + '/' + name_list[i])
			annotation = annotation_.resize(img_size,Image.NEAREST)
			batch_annotation_tensor[i-iteration]  = torch.from_numpy(np.array(annotation)).float()
		# PyTorch use pixel values from 0 to 1
		batch_img_tensor = batch_img_tensor
		batch_annotation_tensor = batch_annotation_tensor / 255
		batch_input = Variable(batch_img_tensor).cuda()
		batch_annotation = Variable(batch_annotation_tensor).cuda()

		
		optimizer.zero_grad()
		output = fcn.forward(batch_input)
		err_seg = criterion(output,batch_annotation)
		err_seg.backward()
		optimizer.step()

inference code:
img = Image.open('...')
size = 256,256
img = img.resize((256,256))
input_tensor = torch.FloatTensor(1,3,256,256)
input_tensor[0] = torch.from_numpy(np.asarray(img))
input_tensor = input_tensor.float()
input_var = Variable(input_tensor)
fcn.eval()
out = fcn.forward(input_var.cuda())

from matplotlib import pyplot as plt

import numpy
out = out.cpu()
out_img = out.data[0]
plt.imshow(out_img.numpy()[0])