if you are using PIL in the getitem function in your Dataset class, you just have to open your mask image that has all the class masks with mode “P.” I am not sure what the advantage of using one-hot encoding-like mask image. I think its better to have one image with all labels drawn in it.
Here is my code that makes mask images.
Loss function should be fine using BCE or categorical cross entropy.
to answer your first question, your should transform your image to tensor and also perform normalization.

Is there a Pytorch function to transform the mask into something readily digestible by the model?


def make_palatte (categories):
	plt_dict = {0: ["backgorund", (0,0,0)]} 

	all_colors_map = []
	palette = [0,0,0]

	for i in tqdm(range(255)):
		for ii in range(255):
			for iii in range(255):
				adding = [i, ii, +iii]
				all_colors_map.append(adding)
	random.shuffle(all_colors_map)

	distance = len(all_colors_map)/(len(categories)+10)  # 10>> buffer
	distance = math.floor(distance)
	
	for idx, one_categ in enumerate(categories):
		id = idx+1
		name = one_categ # word
		color = all_colors_map[(idx+1)*distance]
		palette.extend(color) 
		plt_dict[id] = [name, tuple(color)]  

	return plt_dict, palette


def mask_maker (palette_dict, img_id, height, width, palette, annotation, export_file_path):
	im = Image.new("P", (width, height), color=(0,0,0)) 

	im.putpalette(palette) 

	d =  ImageDraw.Draw(im)
	if len(annotation) == 0: 
		im.save(export_file_path)
		return
	
	#sort
	temp_non_label = []
	new_anno= []
	for anno in annotation:
		if anno["tags"] == "label":
			new_anno.append(anno)
		else:
			#temp_non_label.append(anno)
			pass

	new_anno.extend(temp_non_label)
	
	for region in new_anno: 
		category_id = region["tags"]
	
		
		for a in palette_dict: 
			if palette_dict[a][0] == category_id:
				category_index = a 
				break
			
		xy_tup_list = [] 
		for point in region["points"]:
			x = point["x"]
			y = point["y"]
			xy_tup_list.append((x, y))
		d.polygon(xy_tup_list, fill=category_index) 
		#print(category_id, "\n", export_dir)
#change to however you have your annotation.
	
	#plt.imshow(im)
	#plt.show()
	#print(im.getcolors())
	#print(im.getpalette())
	im.save(export_file_path)