My network takes in a minibatch of 3D MRI image as numpy and should output 1 integer for each image. I currently have n=12 x,y training pairs and n=2 x,y validation pairs.
I was able to get the lowest training MSEloss of 1.4 at epoch 464 with a learning rate of 0.01 and a loss of 1.7 with a learning rate of 0.1  at epoch 90. I’m trying to get a training loss as close to 0 as possible. I tried going from 2 conv layer set to 3 conv layer set. What can I try to do to get a loss closer to 0?
Here’s the model:
class CNNModel(nn.Module):
    def __init__(self):
        super(CNNModel, self).__init__()
        
        self.conv_layer1 = self._conv_layer_set(1, 32)
        self.conv_layer2 = self._conv_layer_set(32,32)
        self.conv_layer3 = self._conv_layer_set(32, 64)

        self.ad_avg_pol=nn.AdaptiveAvgPool3d((1,1,1))
        
        self.fc1=nn.Linear(64,1)
               
        
    def _conv_layer_set(self, in_channels, out_channels):
        conv_layer = nn.Sequential(
        nn.Conv3d(in_channels,    out_channels, kernel_size=(3, 3, 3),stride=1, padding=0),
        nn.LeakyReLU(),
        nn.MaxPool3d((2, 2, 2)) 
        )
        return conv_layer
    
    def forward(self, x):
        x=torch.unsqueeze(x,1) #add dimension for channel
        x = self.conv_layer1(x)
        x = self.conv_layer2(x)
        x = self.conv_layer3(x)

        x= self.ad_avg_pol(x)
        x=x.reshape(x.size(0),-1)
        x=self.fc1(x)
        
        return x