I have designed a DNN which has the following structure:
class my_Net(nn.Module):
    '''
    '''
    def __init__(self, device='cuda:0'):
        super(my_Net, self).__init__()

        self.device = device
        
        
        self.tdmp = TimeDistributed(step = 81)
        
        self.bncnn3drelu = BnCNN3DReLU(input_channels=1, output_channels=32)
        
        self.cnn3drelu_1 = CNN3DReLU(input_channels=32, output_channels=64)
        
        self.cnn3drelu_2 = CNN3DReLU(input_channels=64, output_channels=128)
        
        self.cnn3drelu_4  = CNN3DReLU(input_channels=128, output_channels=256)
                
        self.convlstm = ConvLSTM(input_size=(3, 3), input_channels = 256, hidden_dim=[16, 16, 16],
                 kernel_size=(3, 3),
                 num_layers=3,
                 target=[[80, 2]],
                 bias=True,
                 return_all_layers=False, device = self.device)
#        
        self.cnn2dup_1 = UPCNN2DReLU(size=(3 ,3), input_channels=16, output_channels=1)


    def forward(self, x):
        _ = self.bncnn3drelu(x)
        _ = self.cnn3drelu_1(_)
        _ = self.tdmp(_)
        _ = self.cnn3drelu_2(_)
        _ = self.tdmp(_)
        _ = self.cnn3drelu_3(_)
        _ = self.tdmp(_)
        _ = self.cnn3drelu_4(_)
        _ = self.tdmp(_)
        _ = self.convlstm(_)
        _ = self.cnn2dup_1(_)
        _ = F.interpolate(_, (61,61))
        return _

where TimeDistributed, BnCNN3DReLU, CNN3DReLU, ConvLSTM, and UPCNN2DReLU are"Apply maxpool2d to each fram of time", “batch-normalization+3D CNN + ReLU”, “3D CNN + ReLU”, “convolutional LSTM” and "2D CNN + ReLU +Upsampling ", respectively.
My data is a tensor with batch_size, num_channel, time, heights, width = data.size(). No matter which loss function I have, the gradients are always zero. I’m training the network using the following codes:
out = model(data)

loss = criterion_b(out, label) 
optimizer = optim.Adam(model.parameters(), lr=opt.lr)
optimizer.zero_grad()
loss.backward()
optimizer.step()