Even though I do not save it as class property?
class FCN(nn.Module):

    def __init__(self, num_classes):
        super().__init__()

        feat = list(models.vgg16(pretrained=True).features.children())

        self.feat1 = nn.Sequential(*feat[0:4])
        self.feat2 = nn.Sequential(*feat[5:9])
        self.feat3 = nn.Sequential(*feat[10:16])
        self.feat4 = nn.Sequential(*feat[17:23])
        self.feat5 = nn.Sequential(*feat[24:30])
        self.fconn = nn.Sequential(
            nn.Conv2d(512, 4096, 7),
            nn.ReLU(inplace=True),
            nn.Dropout(),
            nn.Conv2d(4096, 4096, 1),
            nn.ReLU(inplace=True),
            nn.Dropout(),
        )
        self.score_fconn = nn.Conv2d(4096, num_classes, 1)

    def forward(self, x):
        x = self.feat1(x)
        x = self.feat2(x)
        x = self.feat3(x)

        return x


class FCN8(FCN):

 def __init__(self, num_classes):
    super().__init__(num_classes)

    self.score_feat3 = nn.Conv2d(256, num_classes, 1)
    self.score_feat4 = nn.Conv2d(512, num_classes, 1)

def forward(self, x):
    feat3 = super().forward(x)
    feat4 = self.feat4(feat3)
    feat5 = self.feat5(feat4)
    fconn = self.fconn(feat5)

    score_feat3 = self.score_feat3(feat3)
    score_feat4 = self.score_feat4(feat4)
    score_fconn = self.score_fconn(fconn)

    score = F.upsample_bilinear(score_fconn, score_feat4.size()[2:])
    score += score_feat4
    score = F.upsample_bilinear(score, score_feat3.size()[2:])
    score += score_feat3

    return F.upsample_bilinear(score, x.size()[2:])

This however works:
class FCN8(nn.Module):

    def __init__(self, num_classes):
        super().__init__()

        feats = list(models.vgg16(pretrained=True).features.children())

        self.feats = nn.Sequential(*feats[0:9])
        self.feat3 = nn.Sequential(*feats[10:16])
        self.feat4 = nn.Sequential(*feats[17:23])
        self.feat5 = nn.Sequential(*feats[24:30])

        self.fconn = nn.Sequential(
            nn.Conv2d(512, 4096, 7),
            nn.ReLU(inplace=True),
            nn.Dropout(),
            nn.Conv2d(4096, 4096, 1),
            nn.ReLU(inplace=True),
            nn.Dropout(),
        )
        self.score_feat3 = nn.Conv2d(256, num_classes, 1)
        self.score_feat4 = nn.Conv2d(512, num_classes, 1)
        self.score_fconn = nn.Conv2d(4096, num_classes, 1)

    def forward(self, x):
        feats = self.feats(x)
        feat3 = self.feat3(feats)
        feat4 = self.feat4(feat3)
        feat5 = self.feat5(feat4)
        fconn = self.fconn(feat5)

        score_feat3 = self.score_feat3(feat3)
        score_feat4 = self.score_feat4(feat4)
        score_fconn = self.score_fconn(fconn)

        score = F.upsample_bilinear(score_fconn, score_feat4.size()[2:])
        score += score_feat4
        score = F.upsample_bilinear(score, score_feat3.size()[2:])
        score += score_feat3

        return F.upsample_bilinear(score, x.size()[2:])