i get these bad result after a few epochs when the training accuracy becomes ~78%
the data was randomly split into training and validation using sklearn.model_selection.train_test_split method.
This is my Dataset's code with all the pre processing:
class ImageAdapter(metaclass=ABCMeta):
	@abstractmethod
	def get_image(self, index):
		pass

	@abstractmethod
	def get_y(self, index):
		pass

	@abstractmethod
	def __len__(self):
		pass

class FileSystemAdapter(ImageAdapter):
	"""docstring for FileSystemAdapter"""
	def __init__(self, videos_path, dataset_json_filename, width=600, height=600):
		super(FileSystemAdapter, self).__init__()
		self.width = width
		self.height = height
		self.dataset, _ = self.load_dataset(dataset_json_filename)

	def load_dataset(self, dataset_json_filename):
		with open(dataset_json_filename, 'r') as fp:
			data = json.load(fp)
			return data["dataset"], None

	def crop_center(self, img,cropx,cropy):
		y,x,c = img.shape
		startx = x//2 - cropx//2
		starty = y//2 - cropy//2    
		return img[starty:starty+cropy, startx:startx+cropx, :]

	def load_img(self, path):
		img = Image.open(path)
		if img.mode != 'RGB':
			img = img.convert('RGB')
		return img

	def img_to_array(self, img):
		return np.asarray(img, dtype=np.float32)/255.

	def image_loader(self, path):
		img = self.load_img(path)
		img_arr = self.img_to_array(img)
		img_arr = self.crop_center(img_arr, self.width, self.height)
		return img_arr

	def get_y(self, index):
		return self.dataset[index]['target']

	def get_image(self, idx):
		image = self.dataset[idx]['image']
		return self.image_loader(image)

	def __len__(self):
		return len(self.dataset)

class MyDataset(data.Dataset):
    def __init__(self, adapter, spatial_transform=None, normalization=(None, None)):
        self.adapter = adapter
        self.spatial_transform = spatial_transform
        self.mean, self.std = normalization

    def get_y(self, index):
        return self.adapter.get_y(index)

    def __getitem__(self, index):
        image = self.adapter.get_image(index)
        target = self.adapter.get_y(index)

        #normalize image
        if self.mean is not None and self.std is not None:
            image = (image - self.mean)/self.std

        #perform spatial transform
        if self.spatial_transform is not None:
            image = self.spatial_transform(image)

        return image, target

    def __len__(self):
        return len(self.adapter)