<repo_name>jxncyym/SARPN<file_sep>/options/testopt.py
import argparse

def _get_test_opt():
    parser = argparse.ArgumentParser(description = 'Evaluate performance of SARPN on NYU-D v2 test set')
    parser.add_argument('--backbone', default='SENet154', help='select a network as backbone')
    parser.add_argument('--testlist_path', required=True, help='the path of testlist')
    parser.add_argument('--batch_size', type=int, default=1, help='testing batch size')
    parser.add_argument('--root_path', required=True, help="the root path of dataset")
    parser.add_argument('--loadckpt', required=True, help="the path of the loaded model")
    parser.add_argument('--threshold', type=float, default=1.0, help="threshold of the pixels on edges")
    parser.add_argument('--pretrained_dir', type=str, required=True, help="the path of pretrained models")
    # parse arguments
    return parser.parse_args()
<file_sep>/evaluate.py
import time
import torch
import numpy as np
import torch.nn as nn
import torch.nn.parallel
import torch.nn.functional
from utils import *
from models.net import SARPN
from options import get_args
from collections import OrderedDict
from dataloader import nyudv2_dataloader

args = get_args('test')
# lode nyud v2 test set
TestImgLoader = nyudv2_dataloader.getTestingData_NYUDV2(args.batch_size, args.testlist_path, args.root_path)
# model
model = SARPN(args)
model = nn.DataParallel(model)
model.cuda()

# load test model
if args.loadckpt is not None:
    print("loading model {}".format(args.loadckpt))
    state_dict = torch.load(args.loadckpt)
    model.load_state_dict(state_dict)
else:
    print("You have not loaded any models.")

def test():
    model.eval()

    totalNumber = 0

    Ae = 0
    Pe = 0
    Re = 0
    Fe = 0

    errorSum = {'MSE': 0, 'RMSE': 0, 'ABS_REL': 0, 'LG10': 0,
                'MAE': 0,  'DELTA1': 0, 'DELTA2': 0, 'DELTA3': 0}

    for batch_idx, sample in enumerate(TestImgLoader):
        print("Processing the {}th image!".format(batch_idx))
        image, depth  = sample['image'], sample['depth']
        depth = depth.cuda()
        image = image.cuda()

        image = torch.autograd.Variable(image)
        depth = torch.autograd.Variable(depth)
        
        start = time.time()
        pred_depth = model(image)
        end = time.time()
        running_time = end - start
        output = torch.nn.functional.interpolate(pred_depth[-1], size=[depth.size(2), depth.size(3)], mode='bilinear', align_corners=True)

        depth_edge = edge_detection(depth)
        output_edge = edge_detection(output)
        batchSize = depth.size(0)
        totalNumber = totalNumber + batchSize
        errors = evaluateError(output, depth)
        errorSum = addErrors(errorSum, errors, batchSize)
        averageError = averageErrors(errorSum, totalNumber)

        edge1_valid = (depth_edge > args.threshold)
        edge2_valid = (output_edge > args.threshold)

        nvalid = np.sum(torch.eq(edge1_valid, edge2_valid).float().data.cpu().numpy())
        A = nvalid / (depth.size(2)*depth.size(3))

        nvalid2 = np.sum(((edge1_valid + edge2_valid) ==2).float().data.cpu().numpy())
        P = nvalid2 / (np.sum(edge2_valid.data.cpu().numpy()))
        R = nvalid2 / (np.sum(edge1_valid.data.cpu().numpy()))

        F = (2 * P * R) / (P + R)

        Ae += A
        Pe += P
        Re += R
        Fe += F

    Av = Ae / totalNumber
    Pv = Pe / totalNumber
    Rv = Re / totalNumber
    Fv = Fe / totalNumber
    print('PV', Pv)
    print('RV', Rv)
    print('FV', Fv)

    averageError['RMSE'] = np.sqrt(averageError['MSE'])
    print(averageError)


if __name__ == '__main__':
    test()
<file_sep>/train.py
import os
import time
import torch
import numpy as np
import torch.nn as nn
import torch.nn.parallel
import torch.optim as optim
import torch.backends.cudnn as cudnn
from utils import *
from options import get_args
from models.net import SARPN
from torch.autograd import Variable
from tensorboardX import SummaryWriter
from dataloader import nyudv2_dataloader
from models.loss import adjust_gt, total_loss

cudnn.benchmark = True
args = get_args('train')

# Create folder
makedir(args.checkpoint_dir)
makedir(args.logdir)

# creat summary logger
logger = SummaryWriter(args.logdir)

# dataset, dataloader
TrainImgLoader = nyudv2_dataloader.getTrainingData_NYUDV2(args.batch_size, args.trainlist_path, args.root_path)
# model, optimizer
model = SARPN(args)
model = nn.DataParallel(model)
model.cuda()

optimizer = build_optimizer(model = model,
			    learning_rate=args.lr,
			    optimizer_name=args.optimizer_name,
			    weight_decay = args.weight_decay,
			    epsilon=args.epsilon,
			    momentum=args.momentum
			    )

# load parameters
start_epoch = 0
## progress
if args.resume:
	all_saved_ckpts = [ckpt for ckpt in os.listdir(args.checkpoint_dir) if ckpt.endswith(".pth.tar")]
	all_saved_ckpts = sorted(all_saved_ckpts, key=lambda x:int(x.split('_')[-1].split('.'))[0])
	loadckpt = os.path.join(args.checkpoint_dir, all_saved_ckpts[-1])
	start_epoch = all_saved_ckpts[-1].split('_')[-1].split('.')[0]
	print("loading the lastest model in checkpoint_dir: {}".format(loadckpt))
	state_dict = torch.load(loadckpt)
	model.load_state_dict(state_dict)
elif args.loadckpt is not None:
	print("loading model {}".format(args.loadckpt))
	start_epoch = args.loadckpt.split('_')[-1].split('.')[0]
	state_dict = torch.load(args.loadckpt)
	model.load_state_dict(state_dict)
else:
	print("start at epoch {}".format(start_epoch))

## train process
def train():
	for epoch in range(start_epoch, args.epochs):
		adjust_learning_rate(optimizer, epoch, args.lr)
		batch_time = AverageMeter()
		losses = AverageMeter()
		model.train()
		end = time.time()
		for batch_idx, sample in enumerate(TrainImgLoader):
			image, depth = sample['image'], sample['depth']           
			depth = depth.cuda()
			image = image.cuda()
			image = torch.autograd.Variable(image)
			depth = torch.autograd.Variable(depth)			
			optimizer.zero_grad()
			global_step = len(TrainImgLoader) * epoch + batch_idx
			# get the predicted depth maps of different scales
			pred_depth = model(image)
			# adjust ground-truth to the corresponding scales
			gt_depth = adjust_gt(depth, pred_depth)
			# Calculate the total loss 
			loss = total_loss(pred_depth, gt_depth)

			losses.update(loss.item(), image.size(0))
			loss.backward()
			optimizer.step()

			batch_time.update(time.time() - end)
			end = time.time()
			
			if args.do_summary:
				all_draw_image = {"image":image, "pred":pred_depth[-1], "gt":gt_depth[-1]}
				draw_losses(logger, losses.avg, global_step)
				draw_images(logger, all_draw_image, global_step)
			
			#batchSize = depth.size(0)

			print(('Epoch: [{0}][{1}/{2}]\t'
				'Time {batch_time.val:.3f} ({batch_time.sum:.3f})\t'
				'Loss {loss.val:.4f} ({loss.avg:.4f})'
			.format(epoch, batch_idx, len(TrainImgLoader), batch_time=batch_time, loss=losses)))

		if (epoch+1)%1 == 0:
			save_checkpoint(model.state_dict(), filename=args.checkpoint_dir + "SARPN_checkpoints_" + str(epoch + 1) + ".pth.tar")

if __name__ == '__main__':
	train()
<file_sep>/README.md
# Structure-Aware Residual Pyramid Network for Monocular Depth Estimation
This is the implementation of the paper [***Structure-Aware Residual Pyramid Network for Monocular Depth Estimation***](https://arxiv.org/abs/1907.06023), ***IJCAI 2019, <NAME>, <NAME>, and <NAME>.***

## Citation

```
@inproceedings{Chen2019structure-aware,
             title = {Structure-Aware Residual Pyramid Network for Monocular Depth Estimation},
             author = {<NAME> and Chen , Xuejin and <NAME>},
	     conference={International Joint Conferences on Artificial Intelligence},
             year = {2019}   
}

@article{Chen2020LapNet,
  title={Laplacian Pyramid Neural Network for Dense Continuous-Value Regression for Complex Scenes},
  author={<NAME> and <NAME> and Zhang, Yiteng  and <NAME> and <NAME>},
  journal={IEEE TNNLS},
  year={2020}
}
```
## Contents
1. [Introduction](#introduction)<br>
2. [Usage](#usage)<br>
3. [Results](#Results)<br>
4. [Acknowledgements](#Acknowledgements)<br>

## Introduction
Monocular depth estimation is an essential task for scene understanding. The underlying structure of objects and stuff in a complex scene is critical to recovering accurate and visually-pleasing depth maps. Global structure conveys scene layouts,while local structure reflects shape details. Recently developed approaches based on convolutional neural networks (CNNs) significantly improve the performance of depth estimation. However, few of them take into account multi-scale structures in complex scenes. In this paper, we propose a Structure-Aware Residual Pyramid Network (SARPN) to exploit multi-scale structures for accurate depth prediction. We propose a Residual Pyramid Decoder (RPD) which expresses global scene structure in upper levels to represent layouts, and local structure in lower levels to present shape details. At each level, we propose Residual Refinement Modules (RRM) that predict residual maps to progressively add finer structures on the coarser structure predicted at the upper level. In order to fully exploit multi-scale image features, an Adaptive Dense Feature Fusion (ADFF) module, which adaptively fuses effective features from all scales for inferring structures of each scale, is introduced. 
![figure](./images/overview.png)
## Usage
### Dependencies
- [Python3.6](https://www.python.org/downloads/)
- [PyTorch(1.0.1)](https://pytorch.org/)
- [NYU Depth v2](https://cs.nyu.edu/~silberman/datasets/nyu_depth_v2.html)

### Train
As an example, use the following command to train SARPN on NYUDV2.<br>

	CUDA_VISIBLE_DEVICES="0,1,2,3" python train.py --trainlist_path (the path of trainlist(nyu2_train.csv))\
						       --checkpoint_dir (the directory to save the checkpoints)\
						       --root_path (the root path of dataset)\
						       --logdir (the directory to save logs and checkpoints)\
						       --pretrained_dir (the path of pretrained models)\
						       --do_summary
						
### Evaluation
Use the following command to evaluate the trained SARPN on NYUDV2 test data.<br>

	CUDA_VISIBLE_DEVICES="0" python evaluate.py --testlist_path (the path of testlist(nyu2_test.csv))\
						    --root_path (the root path of dataset)\
						    --loadckpt (the path of the loaded model)\
						    --pretrained_dir (the path of pretrained models)\
						    --threshold (threshold of the pixels on edges)

### Pretrained Model
You can download the pretrained model:<br>
**NOTE: You don't need to decompress our pre-trained model. Please use torch.load() to load it.**<br>
[NYUDV2](https://1drv.ms/u/s!AhXIHZfUg-uSaQmwNbyEDywBGMc?e=wjUZwc) [KITTI and Height estimation](https://1drv.ms/u/s!AhXIHZfUg-uSch9ylCA2sBnKpsc?e=dYVfKb)
### Pre-processed Data
You can download the pre-processed data from this [link](https://drive.google.com/file/d/1WoOZOBpOWfmwe7bknWS5PMUCLBPFKTOw/view?usp=sharing), which is shared by [<NAME>.](https://github.com/JunjH/Revisiting_Single_Depth_Estimation)

## Results
**Indoor scene**<br>
![](./images/visualcomparison.png)<br>
**Outdoor scene**<br>
![](./images/comp_kitti.PNG)
## Some examples
![](./examples/demo1.gif)
![](./examples/demo2.gif)

## Acknowledgements
Thanks to <NAME> for opening source of his excellent [work](https://arxiv.org/abs/1803.08673). Our work is inspired by this work and part of codes.
	
<file_sep>/models/modules.py
from collections import OrderedDict
import math
import torch
import torch.nn.functional as F
import torch.nn as nn
from torch.utils import model_zoo
import copy
import numpy as np
from models import senet
from models import resnet
from models import densenet
import matplotlib.image
import matplotlib.pyplot as plt
plt.set_cmap("jet")

class _UpProjection(nn.Sequential):

    def __init__(self, num_input_features, num_output_features):
        super(_UpProjection, self).__init__()

        self.conv1 = nn.Conv2d(num_input_features, num_output_features,
                               kernel_size=5, stride=1, padding=2, bias=False)      
        self.bn1 = nn.BatchNorm2d(num_output_features)
        self.relu = nn.ReLU(inplace=True)
        self.conv1_2 = nn.Conv2d(num_output_features, num_output_features,          
                                 kernel_size=3, stride=1, padding=1, bias=False)
        self.bn1_2 = nn.BatchNorm2d(num_output_features)

        self.conv2 = nn.Conv2d(num_input_features, num_output_features,             
                               kernel_size=5, stride=1, padding=2, bias=False)
        self.bn2 = nn.BatchNorm2d(num_output_features)

    def forward(self, x, size):
        x = F.interpolate(x, size=size, mode='bilinear',align_corners=True)
        x_conv1 = self.relu(self.bn1(self.conv1(x)))
        bran1 = self.bn1_2(self.conv1_2(x_conv1))
        bran2 = self.bn2(self.conv2(x))

        out = self.relu(bran1 + bran2)

        return out

class E_resnet(nn.Module):

    def __init__(self, original_model, num_features = 2048):
        super(E_resnet, self).__init__()        
        self.conv1 = original_model.conv1
        self.bn1 = original_model.bn1
        self.relu = original_model.relu
        self.maxpool = original_model.maxpool

        self.layer1 = original_model.layer1
        self.layer2 = original_model.layer2
        self.layer3 = original_model.layer3
        self.layer4 = original_model.layer4
       

    def forward(self, x):
        x = self.conv1(x)
        x = self.bn1(x)
        x = self.relu(x)
        x_block0 = x

        x = self.maxpool(x)
        x_block1 = self.layer1(x)
        x_block2 = self.layer2(x_block1)
        x_block3 = self.layer3(x_block2)
        x_block4 = self.layer4(x_block3)

        feature_pyramid = [x_block0, x_block1, x_block2, x_block3, x_block4]

        return feature_pyramid

class E_densenet(nn.Module):

    def __init__(self, original_model, num_features = 2208):
        super(E_densenet, self).__init__()        
        self.features = original_model.features

    def forward(self, x):
        x01 = self.features[0](x)
        x02 = self.features[1](x01)
        x03 = self.features[2](x02)
        x_block0 = x03
        x04 = self.features[3](x03)

        x_block1 = self.features[4](x04)
        x_block1 = self.features[5][0](x_block1)
        x_block1 = self.features[5][1](x_block1)
        x_block1 = self.features[5][2](x_block1)
        x_tran1 = self.features[5][3](x_block1)

        x_block2 = self.features[6](x_tran1)
        x_block2 = self.features[7][0](x_block2)
        x_block2 = self.features[7][1](x_block2)
        x_block2 = self.features[7][2](x_block2)
        x_tran2 = self.features[7][3](x_block2)

        x_block3 = self.features[8](x_tran2)
        x_block3 = self.features[9][0](x_block3)
        x_block3 = self.features[9][1](x_block3)
        x_block3 = self.features[9][2](x_block3)
        x_tran3 = self.features[9][3](x_block3)

        x_block4 = self.features[10](x_tran3)
        x_block4 = F.relu(self.features[11](x_block4))

        feature_pyramid = [x_block0, x_block1, x_block2, x_block3, x_block4]

        return feature_pyramid

class E_senet(nn.Module):

    def __init__(self, original_model, num_features = 2048):
        super(E_senet, self).__init__()        
        self.base = nn.Sequential(*list(original_model.children())[:-3])
    
    def forward(self, x):
        x_block0 = nn.Sequential(*list(self.base[0].children())[:-1])(x)      
        x0 = self.base[0](x)       
        x_block1 = self.base[1](x0)                                           
        x_block2 = self.base[2](x_block1)                                     
        x_block3 = self.base[3](x_block2)                                     
        x_block4 = self.base[4](x_block3)                                    
        feature_pyramid = [x_block0, x_block1, x_block2, x_block3, x_block4]
        return feature_pyramid


class Refineblock(nn.Module):
    def __init__(self, num_features, kernel_size):
        super(Refineblock, self).__init__()
        padding=(kernel_size-1)//2

        self.conv1 = nn.Conv2d(1, num_features//2, kernel_size=kernel_size, stride=1, padding=padding, bias=False)

        self.bn1 = nn.BatchNorm2d(num_features//2)

        self.conv2 = nn.Conv2d(  
            num_features//2, num_features//2, kernel_size=kernel_size, stride=1, padding=padding, bias=False)

        self.bn2 = nn.BatchNorm2d(num_features//2)

        self.conv3 = nn.Conv2d(num_features//2, 1, kernel_size=kernel_size, stride=1, padding=padding, bias=True)



    def forward(self, x):
        x_res = self.conv1(x)
        x_res = self.bn1(x_res)
        x_res = F.relu(x_res)
        x_res = self.conv2(x_res)
        x_res = self.bn2(x_res)
        x_res = F.relu(x_res)
        x_res = self.conv3(x_res)

        x2 = x  + x_res
        return x2

# Residual Pyramid Decoder
class RPD(nn.Module):

    def __init__(self, rpd_num_features = 2048, top_num_features=2048):
        super(RPD, self).__init__()


        self.conv = nn.Conv2d(top_num_features, rpd_num_features // 2, kernel_size=1, stride=1, bias=False)                                               
        self.bn = nn.BatchNorm2d(rpd_num_features//2)                                                    

        self.conv5 = nn.Sequential(nn.Conv2d(rpd_num_features, rpd_num_features//2, kernel_size=3, stride=1, padding=1, bias=False),    
                                   nn.BatchNorm2d(rpd_num_features//2),                                                             
                                   nn.ReLU(),                                                                                   
                                   nn.Conv2d(rpd_num_features//2, 1, kernel_size=3, stride=1, padding=1, bias=False))               
        rpd_num_features = rpd_num_features // 2                                                                                              
        self.scale5 = Refineblock(num_features=rpd_num_features, kernel_size=3)                                                     

        rpd_num_features = rpd_num_features // 2                                                                                       

        self.conv4 = nn.Sequential(nn.Conv2d(rpd_num_features, rpd_num_features, kernel_size=3, stride=1, padding=1, bias=False),       
                                   nn.BatchNorm2d(rpd_num_features),                                                                
                                   nn.ReLU(),                                                                                   
                                   nn.Conv2d(rpd_num_features, 1, kernel_size=3, stride=1, padding=1, bias=False))                  

        self.scale4 = Refineblock(num_features=rpd_num_features, kernel_size=3)                                                     

        rpd_num_features = rpd_num_features // 2                                                                                        
        

        self.conv3 = nn.Sequential(nn.Conv2d(rpd_num_features, rpd_num_features, kernel_size=3, stride=1, padding=1, bias=False),       
                                   nn.BatchNorm2d(rpd_num_features),
                                   nn.ReLU(),
                                   nn.Conv2d(rpd_num_features, 1, kernel_size=3, stride=1, padding=1, bias=False))                  

        self.scale3 = Refineblock(num_features=rpd_num_features, kernel_size=3)                                                     

        rpd_num_features = rpd_num_features // 2                                                                                        

        self.conv2 = nn.Sequential(nn.Conv2d(rpd_num_features, rpd_num_features, kernel_size=3, stride=1, padding=1, bias=False),
                                   nn.BatchNorm2d(rpd_num_features),
                                   nn.ReLU(),
                                   nn.Conv2d(rpd_num_features, 1, kernel_size=3, stride=1, padding=1, bias=False))
        self.scale2 = Refineblock(num_features=rpd_num_features, kernel_size=3)                                                     

        rpd_num_features = rpd_num_features // 2                                                                                        

        self.conv1 = nn.Sequential(nn.Conv2d(rpd_num_features, rpd_num_features, kernel_size=3, stride=1, padding=1, bias=False),       
                                   nn.BatchNorm2d(rpd_num_features),                                                                
                                   nn.ReLU(),
                                   nn.Conv2d(rpd_num_features, 1, kernel_size=3, stride=1, padding=1, bias=False))                  

        self.scale1 = Refineblock(num_features=rpd_num_features, kernel_size=3)
    def forward(self, feature_pyramid, fused_feature_pyramid):

        scale1_size = [fused_feature_pyramid[0].size(2), fused_feature_pyramid[0].size(3)]
        scale2_size = [fused_feature_pyramid[1].size(2), fused_feature_pyramid[1].size(3)]
        scale3_size = [fused_feature_pyramid[2].size(2), fused_feature_pyramid[2].size(3)]
        scale4_size = [fused_feature_pyramid[3].size(2), fused_feature_pyramid[3].size(3)]
        scale5_size = [fused_feature_pyramid[4].size(2), fused_feature_pyramid[4].size(3)]
        

        # scale5
        scale5 = torch.cat((F.relu(self.bn(self.conv(feature_pyramid[4]))), fused_feature_pyramid[4]), 1)
        scale5_depth = self.scale5(self.conv5(scale5))

        # scale4
        scale4_res = self.conv4(fused_feature_pyramid[3])
        scale5_upx2 = F.interpolate(scale5_depth, size=scale4_size,
                                    mode='bilinear', align_corners=True)
        scale4_depth = self.scale4(scale4_res + scale5_upx2)

        # scale3 
        scale3_res = self.conv3(fused_feature_pyramid[2])
        scale4_upx2 = F.interpolate(scale4_depth, size=scale3_size,
                                    mode='bilinear', align_corners=True)
        scale3_depth = self.scale3(scale3_res + scale4_upx2)

        # scale2
        scale2_res = self.conv2(fused_feature_pyramid[1])
        scale3_upx2 = F.interpolate(scale3_depth, size=scale2_size,
                                    mode='bilinear', align_corners=True)
        scale2_depth = self.scale2(scale2_res + scale3_upx2)

        # scale1
        scale1_res = self.conv1(fused_feature_pyramid[0])
        scale2_upx2 = F.interpolate(scale2_depth, size=scale1_size,
                                    mode='bilinear', align_corners=True)
        scale1_depth = self.scale1(scale1_res + scale2_upx2)

        scale_depth = [scale5_depth, scale4_depth, scale3_depth, scale2_depth, scale1_depth]

        return scale_depth



# Adaptive Dense Features Fusion module
class ADFF(nn.Module):

    def __init__(self, block_channel, adff_num_features=1280, rpd_num_features=2048):
        super(ADFF, self).__init__()

        rpd_num_features = rpd_num_features // 2
        print("block_channel:", block_channel)
        #scale5
        self.upsample_scale1to5 = _UpProjection(num_input_features=block_channel[0], num_output_features=adff_num_features//5)  
        self.upsample_scale2to5 = _UpProjection(num_input_features=block_channel[1], num_output_features=adff_num_features//5)  
        self.upsample_scale3to5 = _UpProjection(num_input_features=block_channel[2], num_output_features=adff_num_features//5)  
        self.upsample_scale4to5 = _UpProjection(num_input_features=block_channel[3], num_output_features=adff_num_features//5)  
        self.upsample_scale5to5 = _UpProjection(num_input_features=block_channel[4], num_output_features=adff_num_features//5)  
        self.conv_scale5 = nn.Conv2d(adff_num_features, rpd_num_features, kernel_size=3, stride=1, padding=1, bias=False)       # 1280/1024
        self.bn_scale5 = nn.BatchNorm2d(rpd_num_features)

        adff_num_features = adff_num_features // 2                
        rpd_num_features = rpd_num_features // 2           

        # scale4
        self.upsample_scale1to4 = _UpProjection(num_input_features=block_channel[0], num_output_features=adff_num_features//5) 
        self.upsample_scale2to4 = _UpProjection(num_input_features=block_channel[1], num_output_features=adff_num_features//5) 
        self.upsample_scale3to4 = _UpProjection(num_input_features=block_channel[2], num_output_features=adff_num_features//5) 
        self.upsample_scale4to4 = _UpProjection(num_input_features=block_channel[3], num_output_features=adff_num_features//5) 
        self.upsample_scale5to4 = _UpProjection(num_input_features=block_channel[4], num_output_features=adff_num_features//5) 
        self.conv_scale4 = nn.Conv2d(adff_num_features, rpd_num_features, kernel_size=3, stride=1, padding=1, bias=False)       # 640/512
        self.bn_scale4 = nn.BatchNorm2d(rpd_num_features) 

        adff_num_features = adff_num_features // 2                 
        rpd_num_features = rpd_num_features // 2           

        # scale3
        self.upsample_scale1to3 = _UpProjection(num_input_features=block_channel[0], num_output_features=adff_num_features//5)  
        self.upsample_scale2to3 = _UpProjection(num_input_features=block_channel[1], num_output_features=adff_num_features//5)  
        self.upsample_scale3to3 = _UpProjection(num_input_features=block_channel[2], num_output_features=adff_num_features//5)  
        self.upsample_scale4to3 = _UpProjection(num_input_features=block_channel[3], num_output_features=adff_num_features//5)  
        self.upsample_scale5to3 = _UpProjection(num_input_features=block_channel[4], num_output_features=adff_num_features//5)  
        self.conv_scale3 = nn.Conv2d(adff_num_features, rpd_num_features, kernel_size=3, stride=1, padding=1, bias=False)       # 320/256
        self.bn_scale3 = nn.BatchNorm2d(rpd_num_features)

        adff_num_features = adff_num_features // 2                  
        rpd_num_features = rpd_num_features // 2

        # scale2
        self.upsample_scale1to2 = _UpProjection(num_input_features=block_channel[0], num_output_features=adff_num_features//5)  
        self.upsample_scale2to2 = _UpProjection(num_input_features=block_channel[1], num_output_features=adff_num_features//5)  
        self.upsample_scale3to2 = _UpProjection(num_input_features=block_channel[2], num_output_features=adff_num_features//5)  
        self.upsample_scale4to2 = _UpProjection(num_input_features=block_channel[3], num_output_features=adff_num_features//5)  
        self.upsample_scale5to2 = _UpProjection(num_input_features=block_channel[4], num_output_features=adff_num_features//5)  
        self.conv_scale2 = nn.Conv2d(adff_num_features, rpd_num_features, kernel_size=3, stride=1, padding=1, bias=False)        # 160/128
        self.bn_scale2 = nn.BatchNorm2d(rpd_num_features)

        adff_num_features = adff_num_features // 2                  
        rpd_num_features = rpd_num_features // 2                    

        #scale1   
        self.upsample_scale1to1 = _UpProjection(num_input_features=block_channel[0], num_output_features=adff_num_features//5)  
        self.upsample_scale2to1 = _UpProjection(num_input_features=block_channel[1], num_output_features=adff_num_features//5)  
        self.upsample_scale3to1 = _UpProjection(num_input_features=block_channel[2], num_output_features=adff_num_features//5)  
        self.upsample_scale4to1 = _UpProjection(num_input_features=block_channel[3], num_output_features=adff_num_features//5) 
        self.upsample_scale5to1 = _UpProjection(num_input_features=block_channel[4], num_output_features=adff_num_features//5)  
        self.conv_scale1 = nn.Conv2d(adff_num_features, rpd_num_features, kernel_size=3, stride=1, padding=1, bias=False)        # 80/64
        self.bn_scale1 = nn.BatchNorm2d(rpd_num_features)
    
    def forward(self, feature_pyramid):
        scale1_size = [feature_pyramid[0].size(2), feature_pyramid[0].size(3)]
        scale2_size = [feature_pyramid[1].size(2), feature_pyramid[1].size(3)]
        scale3_size = [feature_pyramid[2].size(2), feature_pyramid[2].size(3)]
        scale4_size = [feature_pyramid[3].size(2), feature_pyramid[3].size(3)]
        scale5_size = [feature_pyramid[4].size(2), feature_pyramid[4].size(3)]


        # scale5_mff       8x10
        scale_1to5 = self.upsample_scale1to5(feature_pyramid[0], scale5_size)
        scale_2to5 = self.upsample_scale2to5(feature_pyramid[1], scale5_size)
        scale_3to5 = self.upsample_scale3to5(feature_pyramid[2], scale5_size)
        scale_4to5 = self.upsample_scale4to5(feature_pyramid[3], scale5_size)
        scale_5to5 = self.upsample_scale5to5(feature_pyramid[4], scale5_size)
        scale5_mff = torch.cat((scale_1to5, scale_2to5, scale_3to5, scale_4to5, scale_5to5), 1)
        scale5_mff = F.relu(self.bn_scale5(self.conv_scale5(scale5_mff)))                         

        # scale4_mff       15x19
        scale_1to4 = self.upsample_scale1to4(feature_pyramid[0], scale4_size)
        scale_2to4 = self.upsample_scale2to4(feature_pyramid[1], scale4_size)
        scale_3to4 = self.upsample_scale3to4(feature_pyramid[2], scale4_size)
        scale_4to4 = self.upsample_scale4to4(feature_pyramid[3], scale4_size)
        scale_5to4 = self.upsample_scale5to4(feature_pyramid[4], scale4_size)
        scale4_mff = torch.cat((scale_1to4, scale_2to4, scale_3to4, scale_4to4, scale_5to4), 1)
        scale4_mff = F.relu(self.bn_scale4(self.conv_scale4(scale4_mff)))                        

        # scale3_mff       29x38
        scale_1to3 = self.upsample_scale1to3(feature_pyramid[0], scale3_size)
        scale_2to3 = self.upsample_scale2to3(feature_pyramid[1], scale3_size)
        scale_3to3 = self.upsample_scale3to3(feature_pyramid[2], scale3_size)
        scale_4to3 = self.upsample_scale4to3(feature_pyramid[3], scale3_size)
        scale_5to3 = self.upsample_scale5to3(feature_pyramid[4], scale3_size)
        scale3_mff = torch.cat((scale_1to3, scale_2to3, scale_3to3, scale_4to3, scale_5to3), 1)
        scale3_mff = F.relu(self.bn_scale3(self.conv_scale3(scale3_mff)))                        

        # scale2_mff      57x76
        scale_1to2 = self.upsample_scale1to2(feature_pyramid[0], scale2_size)
        scale_2to2 = self.upsample_scale2to2(feature_pyramid[1], scale2_size)
        scale_3to2 = self.upsample_scale3to2(feature_pyramid[2], scale2_size)
        scale_4to2 = self.upsample_scale4to2(feature_pyramid[3], scale2_size)
        scale_5to2 = self.upsample_scale5to2(feature_pyramid[4], scale2_size)
        scale2_mff = torch.cat((scale_1to2, scale_2to2, scale_3to2, scale_4to2, scale_5to2), 1)
        scale2_mff = F.relu(self.bn_scale2(self.conv_scale2(scale2_mff)))                        

        # scale1_mff      114x152
        scale_1to1 = self.upsample_scale1to1(feature_pyramid[0], scale1_size)             
        scale_2to1 = self.upsample_scale2to1(feature_pyramid[1], scale1_size)
        scale_3to1 = self.upsample_scale3to1(feature_pyramid[2], scale1_size)
        scale_4to1 = self.upsample_scale4to1(feature_pyramid[3], scale1_size)
        scale_5to1 = self.upsample_scale5to1(feature_pyramid[4], scale1_size)
        scale1_mff = torch.cat((scale_1to1, scale_2to1, scale_3to1, scale_4to1, scale_5to1), 1)
        scale1_mff = F.relu(self.bn_scale1(self.conv_scale1(scale1_mff)))                           

        fused_feature_pyramid = [scale1_mff, scale2_mff, scale3_mff, scale4_mff, scale5_mff]

        return fused_feature_pyramid
<file_sep>/dataloader/nyudv2_dataloader.py
import pandas as pd
import numpy as np
from torch.utils.data import Dataset, DataLoader
from torchvision import transforms, utils
from PIL import Image
import random
from dataloader.nyu_transform import *


class NYUDV2Dataset(Dataset):
    """NYUV2D dataset."""

    def __init__(self, csv_file, root_path, transform=None):
        self.frame = pd.read_csv(csv_file, header=None)
        self.transform = transform 
        self.root_path = root_path

    def __getitem__(self, idx):
        image_name = self.frame.ix[idx, 0]
        depth_name = self.frame.ix[idx, 1]
        root_path = self.root_path

        image = Image.open(root_path+image_name)
        depth = Image.open(root_path+depth_name)

        sample = {'image': image, 'depth': depth}

        if self.transform:
            sample = self.transform(sample)

        return sample

    def __len__(self):
        return len(self.frame)


def getTrainingData_NYUDV2(batch_size, trainlist_path, root_path):
    __imagenet_pca = {
        'eigval': torch.Tensor([0.2175, 0.0188, 0.0045]),
        'eigvec': torch.Tensor([
            [-0.5675,  0.7192,  0.4009],
            [-0.5808, -0.0045, -0.8140],
            [-0.5836, -0.6948,  0.4203],
        ])
    }
    __imagenet_stats = {'mean': [0.485, 0.456, 0.406],
                        'std': [0.229, 0.224, 0.225]}

    transformed_training = NYUDV2Dataset(csv_file=trainlist_path,
                                        root_path = root_path,
                                        transform=transforms.Compose([
                                            Scale(240),
                                            RandomHorizontalFlip(),
                                            RandomRotate(5),
                                            CenterCrop([304, 228], [152, 114]),
                                            ToTensor(),
                                            Lighting(0.1, __imagenet_pca[
                                                'eigval'], __imagenet_pca['eigvec']),
                                            ColorJitter(
                                                brightness=0.4,
                                                contrast=0.4,
                                                saturation=0.4,
                                            ),
                                            Normalize(__imagenet_stats['mean'],
                                                      __imagenet_stats['std'])
                                        ]))

    dataloader_training = DataLoader(transformed_training, batch_size,
                                     shuffle=True, num_workers=1, pin_memory=False)

    return dataloader_training


def getTestingData_NYUDV2(batch_size, testlist_path, root_path):

    __imagenet_stats = {'mean': [0.485, 0.456, 0.406],
                        'std': [0.229, 0.224, 0.225]}

    transformed_testing = NYUDV2Dataset(csv_file=testlist_path,
                                        root_path=root_path,
                                        transform=transforms.Compose([
                                           Scale(240),
                                           CenterCrop([304, 228], [304, 228]),
                                           ToTensor(is_test=True),
                                           Normalize(__imagenet_stats['mean'],
                                                     __imagenet_stats['std'])
                                       ]))

    dataloader_testing = DataLoader(transformed_testing, batch_size,
                                    shuffle=False, num_workers=4, pin_memory=False)

    return dataloader_testing
    
<file_sep>/models/net.py
import torch
import torch.nn as nn
import numpy as np
from models import modules
from models import get_models

class SARPN(nn.Module):
	def __init__(self, args):
		super(SARPN, self).__init__()
		print("backbone:", args.backbone)
		self.feature_extraction = get_models(args)

		if args.backbone in ["ResNet18", "ResNet34"]:
			adff_num_features = 640
			rpd_num_features = 512
			block_channel = [64, 64, 128, 256, 512]
			top_num_features = block_channel[-1]

		if args.backbone in ["ResNet50", "ResNet101", "ResNet152"]:
			adff_num_features = 1280
			rpd_num_features = 2048
			block_channel = [64, 256, 512, 1024, 2048]
			top_num_features = block_channel[-1]

		if args.backbone in ["DenseNet121"]:
			adff_num_features = 640
			rpd_num_features = 1024
			block_channel = [64, 128, 256, 512, 1024]
			top_num_features = block_channel[-1]

		if args.backbone in ["DenseNet161"]:
			adff_num_features = 1280
			rpd_num_features = 2048
			block_channel = [96, 192, 384, 1056, 2208]
			top_num_features = block_channel[-1]

		if args.backbone in ["DenseNet169"]:
			adff_num_features = 1280
			rpd_num_features = 2048
			block_channel = [64, 128, 256, 640, 1664]
			top_num_features = block_channel[-1]

		if args.backbone in ["DenseNet201"]:
			adff_num_features = 1280
			rpd_num_features = 2048
			block_channel = [64, 128, 256, 896, 1920]
			top_num_features = block_channel[-1]

		if args.backbone in ["SENet154"]:
			adff_num_features = 1280
			rpd_num_features = 2048
			block_channel = [128, 256, 512, 1024, 2048]
			top_num_features = block_channel[-1]

		if args.backbone in ["SE_ResNet50", "SE_ResNet101", "SE_ResNet152", "SE_ResNext50_32x4d", "SE_ResNext101_32x4d"]:
			adff_num_features = 1280
			rpd_num_features = 2048
			block_channel = [64, 256, 512, 1024, 2048]
			top_num_features = block_channel[-1]

		self.residual_pyramid_decoder = modules.RPD(rpd_num_features, top_num_features)
		self.adaptive_dense_feature_fusion = modules.ADFF(block_channel, adff_num_features, rpd_num_features)

	def forward(self, x):
		feature_pyramid = self.feature_extraction(x)
		fused_feature_pyramid = self.adaptive_dense_feature_fusion(feature_pyramid)
		multiscale_depth = self.residual_pyramid_decoder(feature_pyramid, fused_feature_pyramid)

		return multiscale_depth
