HI Juan ,
Thanks for your reply. The batch size is 2 which means 4 images(3x224x224) to be load in batch.
I have try to the following config on the model :

3 channel -> 0.57MB (3x244x224)
4 channel -> 3.8GB (2x224x224 , 2x224x224)

Here is the code for the model :
class Vgg16PairInput(nn.Module):
    
    def weight_init(self,m):
        classname=m.__class__.__name__
        if classname.find('ConvTran')!=-1:
            m.weight.data.normal_(1,0.5)
    
    def __init__(self):
        super(Vgg16PairInput,self).__init__()
        #self.pretrained_model = models.vgg16(pretrained=True)
        #self.pretrained_model = models.vgg16(pretrained=True)
        self.encoder = nn.Sequential(
                nn.Conv2d(6,64,kernel_size=3,stride=1,padding=1),
                #nn.BatchNorm2d(64),
                nn.ReLU(),
                nn.Conv2d(64,64,kernel_size=3,stride=1,padding=1),
                #nn.BatchNorm2d(64),
                nn.ReLU(),
                nn.MaxPool2d(kernel_size=2, stride=2), #128 112
                nn.Conv2d(64,128,kernel_size=3,stride=1,padding=1),
                #nn.BatchNorm2d(128),
                nn.ReLU(),
                nn.Conv2d(128,128,kernel_size=3,stride=1,padding=1),
                #nn.BatchNorm2d(128),
                nn.ReLU(),
                nn.MaxPool2d(kernel_size=2, stride=2), #64 56
                nn.Conv2d(128,256,kernel_size=3,stride=1,padding=1),
                #nn.BatchNorm2d(256),
                nn.ReLU(),
                nn.Conv2d(256,256,kernel_size=3,stride=1,padding=1),
                #nn.BatchNorm2d(256),
                nn.ReLU(),
                nn.MaxPool2d(kernel_size=2, stride=2), #32 28
                nn.Conv2d(256,512,kernel_size=3,stride=1,padding=1),
                #nn.BatchNorm2d(512),
                nn.ReLU(),
                nn.Conv2d(512,512,kernel_size=3,stride=1,padding=1),
                #nn.BatchNorm2d(512),
                nn.ReLU(),
                nn.Conv2d(512,512,kernel_size=3,stride=1,padding=1),
                #nn.BatchNorm2d(512),
                nn.ReLU(),
                nn.MaxPool2d(kernel_size=2, stride=2), #16 14
                nn.Conv2d(512,512,kernel_size=3,stride=1,padding=1),
                #nn.BatchNorm2d(512),
                nn.ReLU(),
                nn.Conv2d(512,512,kernel_size=3,stride=1,padding=1),
                #nn.BatchNorm2d(512),
                nn.ReLU(),
                nn.Conv2d(512,512,kernel_size=3,stride=1,padding=1),
                #nn.BatchNorm2d(512),
                nn.ReLU(),
                nn.MaxPool2d(kernel_size=2, stride=2), #8  7                    
                                
        )
        
        
        self.classifier=nn.Sequential(
                nn.Linear(7*7*512,4096),
                nn.ReLU(),
                nn.Linear(4096,4096),
                nn.ReLU(),
                nn.Linear(4096,1)
        )
                        
        #del self.pretrained_model
        #self.encoder.apply(self.weight_init)
        
        
    def encode(self,images):
        code=self.encoder(images)
        return code
    
    
    
    def forward(self,leftimage,rightimage):
        combine=torch.cat((leftimage,rightimage),1)
        combine=self.encoder(combine)
        #combine=self.encoder(leftimage)
        
        #print(combine.shape)
        combine=torch.flatten(combine,1)
        #print(combine.shape)
        combine=self.classifier(combine)
        return combine

Thanks,
Dick