I am quite sure now, that it is the dataset because a forward pass of artificially generated data through the same network does not yield the error. I guess it might be somewhere in this snippet of code:
import torch
from torch.utils.data import Dataset
from tools.tools import TensorDict # subclass of dict
import os


class ExtRealMFCCDataset(Dataset):
    """Dataset containing standardized MFCC and phoneme from Tedlium release 2 computed by Kaldi."""

    def __init__(self, kaldi_root=os.environ['KALDI_ROOT']):
        # lots of code to get the data omitted

        # needs to by byte or long tensor to be used as index
        self.phonemes = torch.ByteTensor(phonemes)

        self.normalized_mfcc = torch.tensor(data, requires_grad=True)

    def __len__(self):
        return self.normalized_mfcc.shape[0]

    def __getitem__(self, item):
        phonemes_one_hot = torch.zeros((self.phonemes_range,), requires_grad=True)
        phonemes_one_hot[int(self.phonemes[item])] = 1
        return TensorDict({'mfcc': self.normalized_mfcc[item,:].clone(), 'phoneme': phonemes_one_hot})

Any ideas?