Thanks for the input shapes. I’m unfortunately cannot run the minimal code snippet:
import math
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.utils.model_zoo as model_zoo
import torchvision.models as models

class deeplab_v3_separation(nn.Module):
    def __init__(self):
        super(deeplab_v3_separation, self).__init__()



        deepLabV3ResNet101 = models.segmentation.deeplabv3_resnet101(pretrained=True)
        features_convs_and_ASSP = list(deepLabV3ResNet101.children())

        ResNet101 = models.resnet101(pretrained=True)
        ResNet101_features = list(ResNet101.children())[:-2]

        ASSP_layers = models.segmentation.deeplabv3_resnet101(pretrained=True)
        ASSP_features = list(ASSP_layers.children())

        self.ResNet101_features = nn.Sequential(*ResNet101_features)
        self.ASSP_features = nn.Sequential(*ASSP_features[1])
        self.Semantic_convs = nn.Conv2d(2069, 92, 1)
        self.semanticSoftmax = nn.Softmax2d()
        self.Background_convs = nn.Conv2d(2164, 3, 1)
        self.Separation_convs = nn.Conv2d(2167, 3, 1)
        # self.up = nn.ConvTranspose2d(2069 , 2069, kernel_size=2, stride=2)



    def forward(self, x):
        input_shape = x.shape[-2:]
        x1 = self.ResNet101_features(x)
        x2 = self.ASSP_features(x1)

        x3 = torch.cat((x1,x2),1)
        x3 = F.interpolate(x3, size=input_shape, mode='bilinear', align_corners=False)

        y1 = self.Semantic_convs(x3)
        # y1 = self.semanticSoftmax(y1)
        y2 = self.Background_convs(torch.cat((x,x3,y1),1))
        y3 = self.Separation_convs(torch.cat((x,y1,y2,x3),1))

        return y1, y2, y3

model = deeplab_v3_separation().cuda()
semantic_criterion = nn.CrossEntropyLoss(reduction='none')

x = torch.randn(2, 3, 480, 640).cuda()
semantic_pred, b, c = model(x)
target = torch.randint(0, 92, (*semantic_pred.size())).cuda()

loss = semantic_criterion(semantic_pred, target)
loss.backward()
print(loss)
print(next(model.parameters()).grad)

as my RTX2080Ti (12GB) runs out of memory. Since your RTX2060 has only 6GB of memory, I don’t know how your can execute the code on your device using these shapes.