after first Trainer iterations, model weights become Nan. and I can’t find why …
here is my encoder model:
class ConvBlock(nn.Module):
    def __init__(self, in_channels, out_channels, kernel_size):
        super().__init__()

        self.conv = nn.Conv1d(
            in_channels=in_channels,
            out_channels=out_channels,
            kernel_size=kernel_size,
            padding=kernel_size//2
        )

        self.norm = nn.BatchNorm1d(out_channels)
        self.relu = nn.ReLU()
        self.drop = nn.Dropout()

    def forward(self, x):
        return self.drop(
            self.relu(self.norm(self.conv(x))))

and decoder model:
class ConvDecoder(nn.Module):
    def __init__(self, in_channels, vocab_size):
        super().__init__()
        self.decoder = nn.Conv1d(
            in_channels=in_channels, out_channels=vocab_size, kernel_size=1)

    def forward(self, x):
        return self.decoder(x)


These are simplified version of my model which still make same error
and finally my complete model:
class SpeechRecognitionModel(ModulePT):
    def __init__(self, vocab_size):
        super().__init__()
        self.acoustic_model = ConvBlock(64, 128, 33)
        self.decoder_model = ConvDecoder(
            in_channels=128, vocab_size=vocab_size)
        self.loss = nn.CTCLoss(blank=0)

    def forward(self, x):
        x = self.acoustic_model(x)
        return self.decoder_model(x)

    def configure_optimizers(self):
        return optim.Adamax(self.parameters(), 5e-4)

    def training_step(self, train_batch, *args):
        inputs, input_lengths, outputs, output_lengths = train_batch
        logits = self.forward(inputs)
        logits = logits.permute((-1, 0, 1))
        probs = nn.functional.log_softmax(logits, dim=-1)
        loss = self.loss(probs, outputs, input_lengths, output_lengths)
        self.log("loss", loss)
        return loss
        
asr_model = SpeechRecognitionModel(vocab_size=tokenizer.vocab_size)

where ModulePT is inherite from pl.LightningModule and trained using pl.Trainer on cpu (pl refered to pytorch_lightning).

after first train iteration, my loss function returned nan and then I checked model parameters and this is the result:

for param in asr_model.acoustic_model.parameters():
    print(param)

output:
tensor([[[nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         ...,
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan]],

        [[nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         ...,
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan]],

        [[nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         ...,
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan]],
         ...

I double check my dataloader and dataset which is fine-tuned version of LibriSpeech and became sure non of input sample have any nan value, I ran torch.isnan(inputs).any() for all batch of samples.
what might cause this problem ?