Thanks for the answer! However, I am using already the dilated convolutions and the feature maps still get smaller with each block.
Should dilated convolutions prevent this normally?
Did I implement something wrong?
Here is my implementation: (Note: This is currently still with the upsampling at the end)
class P_Net(nn.Module):
    def __init__(self, in_channels=2, out_channels=16):  # or out_channels = 64
        super(P_Net, self).__init__()

        self.block1 = nn.Sequential(
          nn.Conv3d(in_channels=in_channels, out_channels=out_channels, kernel_size=[3, 3, 3], stride=[1, 1, 1], padding=[0, 0, 0], dilation=[1, 1, 1]),
          nn.ReLU(),
          nn.Conv3d(in_channels=out_channels, out_channels=out_channels, kernel_size=[3, 3, 1], stride=[1, 1, 1], padding=[0, 0, 0], dilation=[1, 1, 1]), # or kernel_size=[3, 3, 3]
          nn.ReLU(),
        )
        self.block2 = nn.Sequential(
          nn.Conv3d(in_channels=out_channels, out_channels=out_channels, kernel_size=[3, 3, 3], stride=[1, 1, 1], padding=[0, 0, 0], dilation=[2, 2, 2]),
          nn.ReLU(),
          nn.Conv3d(in_channels=out_channels, out_channels=out_channels, kernel_size=[3, 3, 1], stride=[1, 1, 1], padding=[0, 0, 0], dilation=[2, 2, 2]), # or kernel_size=[3, 3, 3]
          nn.ReLU(),
        )
        self.block3 = nn.Sequential(
          nn.Conv3d(in_channels=out_channels, out_channels=out_channels, kernel_size=[3, 3, 3], stride=[1, 1, 1], padding=[0, 0, 0], dilation=[3, 3, 3]), # or kernel_size=[3, 3, 1]
          nn.ReLU(),
          nn.Conv3d(in_channels=out_channels, out_channels=out_channels, kernel_size=[3, 3, 1], stride=[1, 1, 1], padding=[0, 0, 0], dilation=[3, 3, 3]),
          nn.ReLU(),
          nn.Conv3d(in_channels=out_channels, out_channels=out_channels, kernel_size=[3, 3, 1], stride=[1, 1, 1], padding=[0, 0, 0], dilation=[3, 3, 3]),
          nn.ReLU(),
        )
        self.block4 = nn.Sequential(
          nn.Conv3d(in_channels=out_channels, out_channels=out_channels, kernel_size=[3, 3, 3], stride=[1, 1, 1], padding=[0, 0, 0], dilation=[4, 4, 4]), # or kernel_size=[3, 3, 1]
          nn.ReLU(),
          nn.Conv3d(in_channels=out_channels, out_channels=out_channels, kernel_size=[3, 3, 1], stride=[1, 1, 1], padding=[0, 0, 0], dilation=[4, 4, 4]),
          nn.ReLU(),
          nn.Conv3d(in_channels=out_channels, out_channels=out_channels, kernel_size=[3, 3, 1], stride=[1, 1, 1], padding=[0, 0, 0], dilation=[4, 4, 4]),
          nn.ReLU(),
        )
        self.block5 = nn.Sequential(
          nn.Conv3d(in_channels=out_channels, out_channels=out_channels, kernel_size=[3, 3, 3], stride=[1, 1, 1], padding=[0, 0, 0], dilation=[5, 5, 5]), # or kernel_size=[3, 3, 1]
          nn.ReLU(),
          nn.Conv3d(in_channels=out_channels, out_channels=out_channels, kernel_size=[3, 3, 1], stride=[1, 1, 1], padding=[0, 0, 0], dilation=[5, 5, 5]),
          nn.ReLU(),
          nn.Conv3d(in_channels=out_channels, out_channels=out_channels, kernel_size=[3, 3, 1], stride=[1, 1, 1], padding=[0, 0, 0], dilation=[5, 5, 5]),
          nn.ReLU(),
        )
        self.block6 = nn.Sequential(
          nn.Conv3d(in_channels=int(out_channels/4)*5, out_channels=out_channels, kernel_size=[1, 1, 1], stride=[1, 1, 1], padding=[0, 0, 0], dilation=[1, 1, 1]), # or kernel_size=[3, 3, 1]
          nn.ReLU(),
          nn.Conv3d(in_channels=out_channels, out_channels=2, kernel_size=[3, 3, 3], stride=[1, 1, 1], padding=[0, 0, 0], dilation=[1, 1, 1]),
          nn.ReLU(),
        )

        self.compress1 = nn.Sequential(
          nn.Conv3d(in_channels=out_channels, out_channels=int(out_channels/4), kernel_size=[1, 1, 1], stride=[1, 1, 1], padding=[0, 0, 0], dilation=[1, 1, 1]),
          nn.ReLU(),
        )
        self.compress2 = nn.Sequential(
          nn.Conv3d(in_channels=out_channels, out_channels=int(out_channels/4), kernel_size=[1, 1, 1], stride=[1, 1, 1], padding=[0, 0, 0], dilation=[1, 1, 1]),
          nn.ReLU(),
        )
        self.compress3 = nn.Sequential(
          nn.Conv3d(in_channels=out_channels, out_channels=int(out_channels/4), kernel_size=[1, 1, 1], stride=[1, 1, 1], padding=[0, 0, 0], dilation=[1, 1, 1]),
          nn.ReLU(),
        )
        self.compress4 = nn.Sequential(
          nn.Conv3d(in_channels=out_channels, out_channels=int(out_channels/4), kernel_size=[1, 1, 1], stride=[1, 1, 1], padding=[0, 0, 0], dilation=[1, 1, 1]),
          nn.ReLU(),
        )
        self.compress5 = nn.Sequential(
          nn.Conv3d(in_channels=out_channels, out_channels=int(out_channels/4), kernel_size=[1, 1, 1], stride=[1, 1, 1], padding=[0, 0, 0], dilation=[1, 1, 1]),
          nn.ReLU(),
        )

        self.upsample1 = nn.Upsample(size=[96, 160, 160], mode='trilinear', align_corners=False)
        self.upsample2 = nn.Upsample(size=[96, 160, 160], mode='trilinear', align_corners=False)
        self.upsample3 = nn.Upsample(size=[96, 160, 160], mode='trilinear', align_corners=False)
        self.upsample4 = nn.Upsample(size=[96, 160, 160], mode='trilinear', align_corners=False)
        self.upsample5 = nn.Upsample(size=[96, 160, 160], mode='trilinear', align_corners=False)
        self.upsample6 = nn.Upsample(size=[96, 160, 160], mode='trilinear', align_corners=False)

    def forward(self, x):
        x = self.block1(x)
        compress1 = self.compress1(x)
        x = self.block2(x)
        compress2 = self.compress2(x)
        x = self.block3(x)
        compress3 = self.compress3(x)
        x = self.block4(x)
        compress4 = self.compress4(x)
        x = self.block5(x)
        compress5 = self.compress5(x)
        compress1 = self.upsample1(compress1)
        compress2 = self.upsample2(compress2)
        compress3 = self.upsample3(compress3)
        compress4 = self.upsample4(compress4)
        compress5 = self.upsample5(compress5)
        x = torch.cat((compress1, compress2, compress3, compress4, compress5), dim=1)
        x = self.block6(x)
        x = self.upsample6(x)
        return x