So I added all these features, and reduced the network even more

class NetUSmall(nn.Module):
    def __init__(self, dtype=torch.float16):
        super(NetUSmall, self).__init__()
        self.n_channels = 1
        self.n_classes = 5
        self.trilinear = True
        self.inc = mp.DoubleConv(1, 64, dtype=dtype)
        self.down1 = mp.Down(64, 128, dtype=dtype)
        self.down2 = mp.Down(128, 128, dtype=dtype)
        self.up1 = mp.Up(256, 128, trilinear=self.trilinear, dtype=dtype)
        self.up2 = mp.Up(192, 64, trilinear=self.trilinear, dtype=dtype)
        self.outc = mp.OutConv(64, self.n_classes, dtype=dtype)

    def forward(self, x):
        x1 = self.inc(x)
        x2 = self.down1(x1)
        x3 = self.down2(x2)
        x = self.up1(x3, x2)
        x = self.up2(x, x1)
        out = self.outc(x)
        return out

This has resulted in only a saving of 100MiB or so. This seems very odd to me. I think the cache is too large - the memory usage peaks during the backward pass but surely there should be more of a reduction?
B