I load PIL.Images as
Dataset:
        with PIL.Image.open(source_path) as fobj:
            source = torch.from_numpy(np.asarray(fobj, dtype=np.float32))
            assert source.shape == torch.Size([1000, 1000, 3])

      # Open the boxes from files and do some processing
      # ......

      # We have not done the labeling yet, there will be 3 probably
      # Type A, Type B and background
      return source, {"boxes": boxes, "labels": torch.ones([N, 1]), dtype=torch,int64)}

Model creation:
    def create_model(self, pretrained=False, num_classes=2):
        """Creates faster rcnn 50 model

        Args:
            pretrained (bool): To use Coco pretrained
            num_classes (int): Number of output classes
        """
        return tv.models.detection.fasterrcnn_resnet50_fpn(
            pretrained=False,
            num_classes=num_classes
        )

Training howerer causes IndexError
        model = self.create_model(
            pretrained=False
        )
        model = model.to(self.device)
        model.train()
        params = [p for p in model.parameters() if p.requires_grad]
        optimizer = torch.optim.SGD(
            params, lr=learning_rate, momentum=.9, weight_decay=5e-4)

        start = time.time()
        for epoch in range(self.epochs):

            for index, (img, targets) in enumerate(dataset):
                img = img.to(self.device)
                for x in targets:
                    targets[x] = targets[x].to(self.device)

                inputs = [img.T] # Transfrom from 1000, 1000, 3 -> 3, 1000, 1000 tensor
                corrects = [targets] # Training payload
                assert targets["boxes"].shape[1] == 4

                output = model(inputs, corrects)
                print(f"Success {index}")
                losses = sum(output.values())

                optimizer.zero_grad()
                losses.backward()
                optimizer.step()

            self.logger.log(
                (
                    f"Epoch {epoch}"
                    f"/{self.epochs} "
                    f"in {self._timestamp(start)} = {losses}"
                )
            )

Traceback (most recent call last):
  File "<string>", line 1, in <module>
  File "<mypath>/ENV/lib/python3.8/site-packages/torch/nn/modules/module.py", line 550, in __call__
    result = self.forward(*input, **kwargs)
  File "<mypath>/ENV/lib/python3.8/site-packages/torchvision/models/detection/generalized_rcnn.py", line 71, in forward
    detections, detector_losses = self.roi_heads(features, proposals, images.image_sizes, targets)
  File "<mypath>/ENV/lib/python3.8/site-packages/torch/nn/modules/module.py", line 550, in __call__
    result = self.forward(*input, **kwargs)
  File "<mypath>/ENV/lib/python3.8/site-packages/torchvision/models/detection/roi_heads.py", line 748, in forward
    proposals, matched_idxs, labels, regression_targets = self.select_training_samples(proposals, targets)
  File "<mypath>/ENV/lib/python3.8/site-packages/torchvision/models/detection/roi_heads.py", line 659, in select_training_samples
    sampled_inds = self.subsample(labels)
  File "<mypath>/ENV/lib/python3.8/site-packages/torchvision/models/detection/roi_heads.py", line 610, in subsample
    sampled_pos_inds, sampled_neg_inds = self.fg_bg_sampler(labels)
  File "<mypath>/ENV/lib/python3.8/site-packages/torchvision/models/detection/_utils.py", line 66, in __call__
    pos_idx_per_image = positive[perm1]
IndexError: index 833 is out of bounds for dimension 0 with size 480

I am completely clueless as the tensor shapes seem to obey the documentation and the indices causing the error seem random ranging usually between 10-500 over the maximum values of the indices. The indices causing the error are also non-deterministic changing randomly with  constant inputs.
>>>inputs[0].shape
torch.Size([3, 1000, 1000])
>>>corrects[0]["boxes"].shape
torch.Size([479, 4])
>>>corrects[0]["labels"].shape
torch.Size([479, 1])
>>>torch.max(corrects[0]["boxes"])
tensor(512.1707)
>>>torch.min(corrects[0]["boxes"])
tensor(-0.2560)

Any help is greatly appreciated.