Looking at this further, I feel like I’m getting some strange dataloader behavior. In my Dataset class in my getitem  I am stacking my 4th dim with np.dstack(depth image) on my RGB image to get a shape of (H,W,4) . The depth images definitely exist, but for some reason I end up with a tensor with 3 channels instead of 4 shortly after starting training. I get no error from the dstack that adds the 4th channel, it just is somehow 3 channels.
python3.6, torch 1.2.0
    def __getitem__(self, i):
        id = self.ids[i]
        image_path = os.path.join(self.images_dir, id)
        dsm_image_path = os.path.join(self.dsm_images_dir, id)
        mask_path = os.path.join(self.masks_dir, id)
        loss_mask_path = os.path.join(self.loss_masks_dir, id) if self.loss_masks_dir else None # loss masks should have same filename as og image

        print(id)
        # concat 3 RGB and 1 Depth channel
        rgb_image = self.read_image(image_path)
        dsm_image = self.read_image(dsm_image_path)
        # resize dsm image to rgb image if sizes are different
      
        image_full = np.dstack([rgb_image, self.normalize_image(dsm_image)])

torch.Size([1, 4, 768, 768])
11946.tiff
train:   0%|                                                                               | 1/63647 [00:08<152:35:49,  8.63s/it, loss_mask: 1.0000, loss: 1.0000, mask_micro_iou: 0.0000]torch.Size([1, 4, 768, 768])
26528.tiff
train:   0%|                                                                                | 2/63647 [00:08<64:36:32,  3.65s/it, loss_mask: 1.0000, loss: 1.0000, mask_micro_iou: 0.0000]torch.Size([1, 4, 768, 768])
100243.tiff
train:   0%|                                                                                | 3/63647 [00:08<36:17:29,  2.05s/it, loss_mask: 1.0000, loss: 1.0000, mask_micro_iou: 0.0000]torch.Size([1, 3, 768, 768])
102467.tiff
train:   0%|                                                                                | 3/63647 [00:09<53:39:15,  3.03s/it, loss_mask: 1.0000, loss: 1.0000, mask_micro_iou: 0.0000]
Traceback (most recent call last):
  File "/usr/lib/python3.6/runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "/usr/lib/python3.6/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/home/bhaktatejas922/internal-geometry-ml/src/train.py", line 188, in <module>
    main(cfg)
  File "/home/bhaktatejas922/internal-geometry-ml/src/train.py", line 176, in main
    **cfg.training.fit,
  File "/home/bhaktatejas922/internal-geometry-ml/src/training/runner.py", line 266, in fit
    output = self._feed_batch(batch)
  File "/home/bhaktatejas922/internal-geometry-ml/src/training/runner.py", line 35, in wrapped
    res = f(*args, **kwargs)
  File "/home/bhaktatejas922/internal-geometry-ml/src/training/runner.py", line 153, in _feed_batch
    output = self.model(*input)
  File "/home/bhaktatejas922/.local/lib/python3.6/site-packages/torch/nn/modules/module.py", line 547, in __call__
    result = self.forward(*input, **kwargs)
  File "/home/bhaktatejas922/.local/lib/python3.6/site-packages/segmentation_models_pytorch/base/model.py", line 15, in forward
    features = self.encoder(x)
  File "/home/bhaktatejas922/.local/lib/python3.6/site-packages/torch/nn/modules/module.py", line 547, in __call__
    result = self.forward(*input, **kwargs)
  File "/home/bhaktatejas922/.local/lib/python3.6/site-packages/segmentation_models_pytorch/encoders/efficientnet.py", line 50, in forward
    x = self._swish(self._bn0(self._conv_stem(x)))
  File "/home/bhaktatejas922/.local/lib/python3.6/site-packages/torch/nn/modules/module.py", line 547, in __call__
    result = self.forward(*input, **kwargs)
  File "/home/bhaktatejas922/.local/lib/python3.6/site-packages/efficientnet_pytorch/utils.py", line 271, in forward
    x = F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)
RuntimeError: Given groups=1, weight of size 48 4 3 3, expected input[1, 3, 770, 770] to have 4 channels, but got 3 channels instead