@ptrblck Thanks for reply!
Now that I have inspected the model I wonder if there is a problem how I initialize some layers Namely b1, b2, b3, b4,b5.
class JModel(nn.Module):
    def __init__(self, b=1, r=5):
        super(Jasper, self).__init__()

        self.conv1 = ConvProc(64, 256, 11, 0.2)
        self.b1 = [Block(inc, outc, 11, 0.2, r) for inc, outc in get_sizes(256, 256, b)]
        self.b2 = [Block(inc, outc, 13, 0.2, r) for inc, outc, in get_sizes(256, 384, b)]
        self.b3 = [Block(inc, outc, 17, 0.3, r) for inc, outc, in get_sizes(384, 512, b)]
        self.b4 = [Block(inc, outc, 21, 0.3, r) for inc, outc, in get_sizes(512, 640, b)]
        self.b5 = [Block(inc, outc, 25, 0.3, r) for inc, outc, in get_sizes(640, 768, b)]
        self.blocks = self.b1 + self.b2 + self.b3 + self.b4 + self.b5
        self.conv2 = ConvProc(768, 896, 29, 0.4, dilation=2)
        self.conv3 = ConvProc(896, 1024, 29, 0.4)
        self.conv4 = nn.Conv1d(1024, 28, 1)

    def forward(self, x):
        feats = self.conv1(x)
        for block in self.blocks:
            feats = block(feats)
        feats = self.conv2(feats)
        feats = self.conv3(feats)
        feats = self.conv4(feats)
        return feats

EDIT: now that I’ve looked it up it seems I need nn.ModuleList?
EDIT2: It works with nn.ModuleList
Thanks for pointing in the right direction!