Hi @ptrblck ,
thanks a lot for your reply.
Here is my dataloader code :
import os
import collections
import torch
import torchvision
import numpy as np
import scipy.misc as m
import matplotlib.pyplot as plt
import yaml
from PIL import Image
from torch.utils import data


class ImageLoader(data.Dataset):
    
    
    def make_dataset(self,dir, set):
        images = []
        if set == 'train':
            fname = os.path.join(dir, 'train_test_split.yaml')
        elif set == 'test':
            fname = os.path.join(dir, 'train_test_split.yaml')

        # read the content of the file
        with open(fname,'r') as f:
            doc = yaml.load(f)

        imagesNum = doc[set]
        imageFolderPath = dir + os.sep + 'images' + os.sep
        file_list = []
        
        for x in imagesNum:
            item = imageFolderPath + os.sep + str(x)+'_image.png'

            width, height = Image.open(open(item,'rb')).size
            #print("Width ==> ",width," Height ==> ",height)
            
            file_list.append(item)
        
        self.files[set]=file_list
            
        return
    
    def __init__(self, root, split="train", img_size=None,input_transform=None, target_transform=None):
        self.root = root
        self.split = split
        self.img_size = [1296, 966]
        
        
        self.n_classes = 2
        self.files = collections.defaultdict(list)
        
        self.input_transform = input_transform
        self.target_transform = target_transform
        
        self.make_dataset(root,split)

    def __len__(self):
        return len(self.files[self.split])

    def __getitem__(self, index):
        img_name = self.files[self.split][index]
                
        img_path = img_name
        img_name = os.path.split(img_path)[1]
        #print("Img name ==> ",img_name)
        img_num = img_name.split("_")[0]
        #print("Img num ==> ",img_num)
        
        lbl_path = self.root + os.sep + 'annotations' + os.sep + img_num +"_" + "annotation.png"

        with open(img_path, 'rb') as f:
            image = Image.open(f).convert('RGB')
            
        
        with open(lbl_path, 'rb') as f:
            label = Image.open(f).convert('P')
            
        if self.input_transform is not None:
            image = self.input_transform(image)
        if self.target_transform is not None:
            label = self.target_transform(label) 

        return image, label

and I am transforming the images before passing them to model :

import numpy as np
import torch

from PIL import Image

def colormap(n):
    cmap=np.zeros([n, 3]).astype(np.uint8)

    for i in np.arange(n):
        r, g, b = np.zeros(3)

        for j in np.arange(8):
            r = r + (1<<(7-j))*((i&(1<<(3*j))) >> (3*j))
            g = g + (1<<(7-j))*((i&(1<<(3*j+1))) >> (3*j+1))
            b = b + (1<<(7-j))*((i&(1<<(3*j+2))) >> (3*j+2))

        cmap[i,:] = np.array([r, g, b])

    return cmap

class Relabel:

    def __init__(self, olabel, nlabel):
        self.olabel = olabel
        self.nlabel = nlabel

    def __call__(self, tensor):
        assert isinstance(tensor, torch.LongTensor), 'tensor needs to be LongTensor'
        tensor[tensor == self.olabel] = self.nlabel
        return tensor


class ToLabel:

    def __call__(self, image):
        return torch.from_numpy(np.array(image)).long().unsqueeze(0)


class Colorize:

    def __init__(self, n=22):
        self.cmap = colormap(256)
        self.cmap[n] = self.cmap[-1]
        self.cmap = torch.from_numpy(self.cmap[:n])

    def __call__(self, gray_image):
        size = gray_image.size()
        color_image = torch.ByteTensor(3, size[1], size[2]).fill_(0)

        for label in range(1, len(self.cmap)):
            mask = gray_image[0] == label

            color_image[0][mask] = self.cmap[label][0]
            color_image[1][mask] = self.cmap[label][1]
            color_image[2][mask] = self.cmap[label][2]

        return color_image

and in the following way I am calling this transformation functions
color_transform = Colorize()
image_transform = ToPILImage()

input_transform = Compose([
    CenterCrop(256),
    ToTensor(),
    Normalize([.485, .456, .406], [.229, .224, .225]),
])
target_transform = Compose([
    CenterCrop(256),
    ToLabel(),
    Relabel(255, 2),
])

I am suspecting the erroneous labelling is occurring due to target_transform. But , I am not able to figure out exact place .
Could you please help ?
Thanks a lot in advance