So I managed to increase the batch size to 4 by reducing the number of filters in the first layer:
class NetU(nn.Module):
    ''' U-Net code.'''

    def __init__(self, dtype=torch.float16):
        super(NetU, self).__init__()
        self.n_channels = 1
        self.n_classes = 5
        self.trilinear = True
        self.inc = mp.DoubleConv(1, 32, dtype=dtype)
        self.down1 = mp.Down(32, 64, dtype=dtype)
        self.down2 = mp.Down(64, 128, dtype=dtype)
        self.down3 = mp.Down(128, 256, dtype=dtype)
        self.up1 = mp.Up(384, 128, trilinear=self.trilinear, dtype=dtype)
        self.up2 = mp.Up(192, 64, trilinear=self.trilinear, dtype=dtype)
        self.up3 = mp.Up(96, 32, trilinear=self.trilinear, dtype=dtype)
        self.outc = mp.OutConv(32, self.n_classes, dtype=dtype)

    def forward(self, x):
        x1 = self.inc(x)
        x2 = self.down1(x1)
        x3 = self.down2(x2)
        x4 = self.down3(x3)
        x = self.up1(x4, x3)
        x = self.up2(x, x2)
        x = self.up3(x, x1)
        out = self.outc(x)
        return out

Unfortunately, it’s still only a minor increase. The memory seems to hover around 20% of the total which should mean I have a fair bit of headspace, but there appears to be a blip early in the training loop where memory spikes quite highly. Not sure what is going on there. I’ll investigate a little more.
B