Sorry I don’t know how to reproduce it with random tensors, but this is exactly what I did (it’s just the training part):
(It won’t give any error and works smoothly but with just one gpu)
device = torch.device("cuda")
device_ids = [i for i in range(torch.cuda.device_count())]

model = torchvision.models.detection.maskrcnn_resnet50_fpn(pretrained=True)
in_features = model.roi_heads.box_predictor.cls_score.in_features
in_features_mask = model.roi_heads.mask_predictor.conv5_mask.in_channels
model.roi_heads.mask_predictor = MaskRCNNPredictor(in_features_mask,
													256,
													num_classes)	
params = [p for p in model.parameters() if p.requires_grad]
optimizer = torch.optim.SGD(params, lr=0.005, momentum=0.9, weight_decay=0.0005)
lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=3, gamma=0.1)


model=  torch.nn.DataParallel(model, device_ids=[0, 1, 2])
model.to(device)
model =	model.cuda()

train_data = get_coco(path_to_data, "train", None)  

train_loader = torch.utils.data.DataLoader( train_data 
												,batch_size=batch_size
												,num_workers=num_workers
												,collate_fn=collate_fn
											)
											
											
for epoch in range(num_epochs):
		
		model.train()	
		bj = 0	

		for images, targets in train_loader:	

			images = list(torchvision.transforms.ToTensor()(image).to(device) for image in images)
			
			targets = [{k: v.to(device) for k, v in t.items()} for t in targets]
			
			loss_dict = model(images, targets)
			
			losses = sum(loss for loss in loss_dict.values())
		
			# reduce losses over all GPUs for logging purposes
			loss_dict_reduced = utils.reduce_dict(loss_dict)
			losses_reduced = sum(loss for loss in loss_dict_reduced.values())
		
			loss_value = losses_reduced.item()
		
			if not math.isfinite(loss_value):
				print("Loss is {}, stopping training".format(loss_value))
				print(loss_dict_reduced)
				sys.exit(1)
		
			optimizer.zero_grad()
			losses.backward()
			optimizer.step()

Thanks.