<file_sep>//
// Created by lshi on 17-9-25.
//

//#include "deform_conv3d_cuda.h"
#include "TH/TH.h"
#include "THC/THC.h"
#include "deform_conv3d_cuda_kernel.h"
#include "iostream"

using namespace std;
extern THCState *state;

inline int input_to_output(int input, int pad, int kernel, int stride) {
    return (input + 2 * pad - kernel) / stride + 1;
}

void check(bool e, const char *msg) {
    if (!e)
        THError(msg);
}

void shape_check(THCState *state,
                 THCudaDoubleTensor *input, THCudaDoubleTensor *weight, THCudaDoubleTensor *offset, THCudaDoubleTensor *output,
                 THCudaDoubleTensor *columns,
                 const int pad_l, const int pad_h, const int pad_w,
                 const int stride_l, const int stride_h, const int stride_w,
                 const int channel_per_deformable_group) {
    THCAssertSameGPU(THCudaDoubleTensor_checkGPU(state, 5, input, weight, offset, output, columns));

    int kernel_dim = weight->nDimension;
    check(kernel_dim == 5, "kernel dim");
    long kernel_output_c = weight->size[0];
    long kernel_input_c = weight->size[1];
    long kernel_l = weight->size[2];
    long kernel_h = weight->size[3];
    long kernel_w = weight->size[4];

    int input_dim = input->nDimension;
    check(input_dim == 5, "input dim");
    long input_b = input->size[0];
    long input_c = input->size[1];
    check(input_c == kernel_input_c, "input c != kernel c");
    long input_l = input->size[2];
    long input_h = input->size[3];
    long input_w = input->size[4];

    //out NC"L"H"W"
    int output_dim = output->nDimension;
    check(output_dim == 5, "output dim");
    long output_b = output->size[0];
    check(output_b == input_b, "output b != input b");
    long output_c = output->size[1];
    check(output_c == kernel_output_c, "ou c != ker c");
    long output_l = output->size[2];
    long output_h = output->size[3];
    long output_w = output->size[4];

    int offset_dim = offset->nDimension;
    check(offset_dim == 5, "off dim");
    long offset_b = offset->size[0];
    check(offset_b == input_b, "off batch");
    long offset_c = offset->size[1];
//    cout<<offset_c<<" "<<input_c / channel_per_deformable_group * 3 * kernel_l * kernel_h * kernel_w<<endl;
    check(offset_c == input_c / channel_per_deformable_group, "off channel");
    long offset_l = offset->size[2];
    check(offset_l == output_l, "off l");
    long offset_h = offset->size[3];
    check(offset_h == output_h, "off h");
    long offset_w = offset->size[4];
    check(offset_w == output_w, "off w");

    int columns_dim = columns->nDimension;
    check(columns_dim == 2, "columns dim");
    long columns_row = columns->size[0];
    check(columns_row == input_c * kernel_l * kernel_h * kernel_w, "columns row");
    long columns_col = columns->size[1];
    check(columns_col == output_l * output_h * output_w, "columns col");

    long output_ll = input_to_output(input_l, pad_l, kernel_l, stride_l);
    check(output_ll == output_l, "error output ll");
    long output_hh = input_to_output(input_h, pad_h, kernel_h, stride_h);
    check(output_hh == output_h, "error outhh");
    long output_ww = input_to_output(input_w, pad_w, kernel_w, stride_w);
    check(output_ww == output_w, "error outww");
}


int deform_conv_forward_cuda(
        THCudaDoubleTensor *input, THCudaDoubleTensor *weight, THCudaDoubleTensor *offset,
        THCudaDoubleTensor *columns, THCudaDoubleTensor *output,
        const int pad_l, const int pad_h, const int pad_w,
        const int stride_l, const int stride_h, const int stride_w,
        const int channel_per_deformable_group) {
//    cout<<output->size[0]<<output->size[1]<<output->size[2];
    shape_check(state, input, weight, offset, output, columns,
                pad_l, pad_h, pad_w, stride_l, stride_h, stride_w, channel_per_deformable_group);

    THCudaDoubleTensor *input_n = THCudaDoubleTensor_new(state);
    THCudaDoubleTensor *offset_n = THCudaDoubleTensor_new(state);
    THCudaDoubleTensor *output_n = THCudaDoubleTensor_new(state);

    for(int i=0;i<input->size[0];i++){
        THCudaDoubleTensor_select(state, input_n, input, 0, i);
        THCudaDoubleTensor_select(state, offset_n, offset, 0, i);
        THCudaDoubleTensor_select(state, output_n, output, 0, i);

        deformable_im2col(THCState_getCurrentStream(state),
                          THCudaDoubleTensor_data(state, input_n), THCudaDoubleTensor_data(state, offset_n),
                          input->size[1], input->size[2], input->size[3], input->size[4],
                          output->size[2], output->size[3], output->size[4],
                          weight->size[2], weight->size[3], weight->size[4],
                          pad_l, pad_h, pad_w,
                          stride_l, stride_h, stride_w,
                          channel_per_deformable_group, THCudaDoubleTensor_data(state, columns));
        //GEMM(TRANSA,TRANSB,M,N,K,ALPHA,A,LDA,B,LDB,BETA,C,LDC)
        //C := alpha*op( A )*op( B ) + beta*C,
        int m = weight->size[0];
        int k = columns->size[0];
        int n = columns->size[1];

        THCudaBlas_Dgemm(state, 'n', 'n', n, m, k,
                         1.0f, THCudaDoubleTensor_data(state, columns), n,
                         THCudaDoubleTensor_data(state, weight), k,
                         0.0f, THCudaDoubleTensor_data(state, output_n), n);
        }

    THCudaDoubleTensor_free(state, input_n);
    THCudaDoubleTensor_free(state, offset_n);
    THCudaDoubleTensor_free(state, output_n);

    return 1;
}


int deform_conv_backward_input_offset_cuda(
        THCudaDoubleTensor *input, THCudaDoubleTensor *weight, THCudaDoubleTensor *offset, THCudaDoubleTensor *grad_output,
        THCudaDoubleTensor *columns, THCudaDoubleTensor *grad_input, THCudaDoubleTensor *grad_offset,
        const int pad_l, const int pad_h, const int pad_w,
        const int stride_l, const int stride_h, const int stride_w,
        const int channel_per_deformable_group) {
    shape_check(state, input, weight, grad_offset, grad_output, columns,
                pad_l, pad_h, pad_w, stride_l, stride_h, stride_w, channel_per_deformable_group);
    check(THCudaDoubleTensor_isSameSizeAs(state, offset, grad_offset), "offset vs grad_offset");
    THCAssertSameGPU(THCudaDoubleTensor_checkGPU(state, 2, offset, grad_offset));

    THCudaDoubleTensor *input_n = THCudaDoubleTensor_new(state);
    THCudaDoubleTensor *offset_n = THCudaDoubleTensor_new(state);
    THCudaDoubleTensor *grad_input_n = THCudaDoubleTensor_new(state);
    THCudaDoubleTensor *grad_offset_n = THCudaDoubleTensor_new(state);
    THCudaDoubleTensor *grad_output_n = THCudaDoubleTensor_new(state);

    for(int i=0;i<input->size[0];i++){
        THCudaDoubleTensor_select(state, input_n, input, 0, i);
        THCudaDoubleTensor_select(state, offset_n, offset, 0, i);
        THCudaDoubleTensor_select(state, grad_input_n, grad_input, 0, i);
        THCudaDoubleTensor_select(state, grad_offset_n, grad_offset, 0, i);
        THCudaDoubleTensor_select(state, grad_output_n, grad_output, 0, i);

    //Wt * O = C
    long m = columns->size[0];
    long n = columns->size[1];
    long k = weight->size[0];
    THCudaBlas_Dgemm(state, 'n', 't', n, m, k,
                     1.0f, THCudaDoubleTensor_data(state, grad_output_n), n,
                     THCudaDoubleTensor_data(state, weight), m,
                     0.0f, THCudaDoubleTensor_data(state, columns), n);
    deformable_col2im_offset(THCState_getCurrentStream(state), THCudaDoubleTensor_data(state, columns),
                             THCudaDoubleTensor_data(state, input_n), THCudaDoubleTensor_data(state, offset_n),
                             input->size[1], input->size[2], input->size[3], input->size[4],
                             grad_output->size[2], grad_output->size[3], grad_output->size[4],
                             weight->size[2], weight->size[3], weight->size[4],
                             pad_l, pad_h, pad_w,
                             stride_l, stride_h, stride_w,
                             channel_per_deformable_group, THCudaDoubleTensor_data(state, grad_offset_n));

    deformable_col2im_input(THCState_getCurrentStream(state),
                                THCudaDoubleTensor_data(state, columns),
                                THCudaDoubleTensor_data(state, offset_n),
                                grad_input->size[1], grad_input->size[2], grad_input->size[3], grad_input->size[4],
                                grad_output->size[2], grad_output->size[3], grad_output->size[4],
                                weight->size[2], weight->size[3], weight->size[4],
                                pad_l, pad_h, pad_w,
                                stride_l, stride_h, stride_w, channel_per_deformable_group,
                                THCudaDoubleTensor_data(state, grad_input_n));

    }

    THCudaDoubleTensor_free(state, input_n);
    THCudaDoubleTensor_free(state, offset_n);
    THCudaDoubleTensor_free(state, grad_input_n);
    THCudaDoubleTensor_free(state, grad_offset_n);
    THCudaDoubleTensor_free(state, grad_output_n);

    return 1;
}


int deform_conv_backward_weight_cuda(
        THCudaDoubleTensor *input, THCudaDoubleTensor *offset, THCudaDoubleTensor *grad_output,
        THCudaDoubleTensor *columns, THCudaDoubleTensor *grad_weight,
        const int pad_l, const int pad_h, const int pad_w,
        const int stride_l, const int stride_h, const int stride_w,
        const int channel_per_deformable_group) {
    shape_check(state, input, grad_weight, offset, grad_output, columns,
                pad_l, pad_h, pad_w, stride_l, stride_h, stride_w, channel_per_deformable_group);
    THCudaDoubleTensor *input_n = THCudaDoubleTensor_new(state);
    THCudaDoubleTensor *offset_n = THCudaDoubleTensor_new(state);
    THCudaDoubleTensor *grad_output_n = THCudaDoubleTensor_new(state);

    for(int i=0;i<input->size[0];i++){
        THCudaDoubleTensor_select(state, input_n, input, 0, i);
        THCudaDoubleTensor_select(state, offset_n, offset, 0, i);
        THCudaDoubleTensor_select(state, grad_output_n, grad_output, 0, i);

        deformable_im2col(THCState_getCurrentStream(state),
                          THCudaDoubleTensor_data(state, input_n), THCudaDoubleTensor_data(state, offset_n),
                          input->size[1], input->size[2], input->size[3], input->size[4],
                          grad_output->size[2], grad_output->size[3], grad_output->size[4],
                          grad_weight->size[2], grad_weight->size[3], grad_weight->size[4],
                          pad_l, pad_h, pad_w,
                          stride_l, stride_h, stride_w,
                          channel_per_deformable_group,
                          THCudaDoubleTensor_data(state, columns));
        //GEMM(TRANSA,TRANSB,M,N,K,ALPHA,A,LDA,B,LDB,BETA,C,LDC)
        //C := alpha*op( A )*op( B ) + beta*C,
        int m = grad_weight->size[0];
        int k = columns->size[1];
        int n = columns->size[0];
        THCudaBlas_Dgemm(state, 't', 'n', n, m, k,
                         1.0f, THCudaDoubleTensor_data(state, columns), k,
                         THCudaDoubleTensor_data(state, grad_output_n), k,
                         1.0f, THCudaDoubleTensor_data(state, grad_weight), n);
    }

    THCudaDoubleTensor_free(state, input_n);
    THCudaDoubleTensor_free(state, offset_n);
    THCudaDoubleTensor_free(state, grad_output_n);

    return 1;
}
<file_sep>import torch
from deform_conv2d_functions import ConvOffset2dFunction
from torch.autograd import Variable
import os
from gradcheck import gradcheck

os.environ['CUDA_VISIBLE_DEVICES'] = '3'
batchsize = 2
c_in = 2
c_out = 3
inpu = 7
kernel = 1
stri = 2
pad = 1
out = int((inpu + 2 * pad - kernel) / stri + 1)
channel_per_group = 2
g_off = c_in // channel_per_group
c_off = g_off * kernel * kernel * 2

# s = Variable(torch.rand(batchsize,2,3).type(torch.DoubleTensor).cuda(), requires_grad=True)
# x = Variable(torch.rand(batchsize, c_in, inpu, inpu).type(torch.DoubleTensor).cuda(), requires_grad=True)
# def stn(s, x):
#     g = F.affine_grid(s, x.size())
#     x = F.grid_sample(x, g)
#     return x
# print(gradcheck(stn, (s, x)))


# conv_offset2d = ConvOffset2d(c_in, c_out, kernel, stri, pad, channel_per_group).cuda()
conv_offset2d = ConvOffset2dFunction((stri, stri), (pad, pad), channel_per_group)
# conv_offset2d = ConvOffset2d(c_in, c_out, kernel, stri, pad, channel_per_group).cuda()

# inputs = Variable(torch.DoubleTensor([[[[1., 1., 1, 1, 1]] * 5] * c_in] * batchsize).type(torch.DoubleTensor).cuda(),
#                   requires_grad=True)
# offsets = Variable(torch.DoubleTensor([[[[0.001] * out] * out,
#                                        [[0.001] * out] * out]
#                                       * kernel * kernel * g_off] * batchsize).type(torch.DoubleTensor).cuda(),
#                    requires_grad=False)
# weight = Variable(torch.ones(c_out, c_in, kernel, kernel).type(torch.DoubleTensor).cuda(),
#                   requires_grad=False)
#
inputs = Variable(torch.rand(batchsize, c_in, inpu, inpu).double().cuda(), requires_grad=True)
offsets = Variable(torch.rand(batchsize, c_off, out, out).double().cuda(),
                   requires_grad=True)
weight = Variable(torch.rand(c_out, c_in, kernel, kernel).double().cuda(),
                  requires_grad=True)

print(gradcheck(conv_offset2d, (inputs, offsets, weight)))
<file_sep>//
// Created by lshi on 17-9-25.
//

//#ifndef DEFORM3D_NEW_PYTORCH_DEFORM_CONV3D_CUDA_BACKWARD_CU_H
//#define DEFORM3D_NEW_PYTORCH_DEFORM_CONV3D_CUDA_BACKWARD_CU_H

//#include "THC/THC.h"
//#include "TH/TH.h"

//template<typename DType>
void deformable_im2col(cudaStream_t stream,
                       const double *data_in, const double *data_offset,
                       const int input_c,
                       const int input_l, const int input_h, const int input_w,
                       const int output_l, const int output_h, const int output_w,
                       const int kernel_l, const int kernel_h, const int kernel_w,
                       const int pad_l, const int pad_h, const int pad_w,
                       const int stride_l, const int stride_h, const int stride_w,
                       const int channel_per_deformable_group, double *data_col);


//template<typename DType>
void deformable_col2im_input(cudaStream_t stream,
                             const double *data_col, const double *data_offset,
                             const int input_c,
                             const int input_l, const int input_h, const int input_w,
                             const int output_l, const int output_h, const int output_w,
                             const int kernel_l, const int kernel_h, const int kernel_w,
                             const int pad_l, const int pad_h, const int pad_w,
                             const int stride_l, const int stride_h, const int stride_w,
                             const int channel_per_deformable_group, double *grad_im);

//template<typename DType>
void deformable_col2im_offset(cudaStream_t stream, const double *data_col,
                              const double *data_im, const double *data_offset,
                              const int input_c,
                              const int input_l, const int input_h, const int input_w,
                              const int output_l, const int output_h, const int output_w,
                              const int kernel_l, const int kernel_h, const int kernel_w,
                              const int pad_l, const int pad_h, const int pad_w,
                              const int stride_l, const int stride_h, const int stride_w,
                              const int channel_per_deformable_group,
                              double *grad_offset);

//#endif //DEFORM3D_NEW_PYTORCH_DEFORM_CONV3D_CUDA_BACKWARD_CU_H
<file_sep>//
// Created by lshi on 17-9-25.
//

//#ifndef DEFORM3D_NEW_PYTORCH_DEFORM_CONV3D_CUDA_H
//#define DEFORM3D_NEW_PYTORCH_DEFORM_CONV3D_CUDA_H

//#include "TH/TH.h"
//#include "THC/THC.h"
//#include "deform_conv3d_cuda_kernel.h"

int deform_conv_forward_cuda(
        THCudaDoubleTensor *input, THCudaDoubleTensor *weight, THCudaDoubleTensor *offset,
        THCudaDoubleTensor *columns, THCudaDoubleTensor *output,
        const int pad_l, const int pad_h, const int pad_w,
        const int stride_l, const int stride_h, const int stride_w,
        const int channel_per_deformable_group);

int deform_conv_backward_input_offset_cuda(
        THCudaDoubleTensor *input, THCudaDoubleTensor *weight, THCudaDoubleTensor *offset, THCudaDoubleTensor *grad_output,
        THCudaDoubleTensor *columns, THCudaDoubleTensor *grad_input, THCudaDoubleTensor *grad_offset,
        const int pad_l, const int pad_h, const int pad_w,
        const int stride_l, const int stride_h, const int stride_w,
        const int channel_per_deformable_group);

int deform_conv_backward_weight_cuda(
        THCudaDoubleTensor *input, THCudaDoubleTensor *offset, THCudaDoubleTensor *grad_output,
        THCudaDoubleTensor *columns, THCudaDoubleTensor *grad_weight,
        const int pad_l, const int pad_h, const int pad_w,
        const int stride_l, const int stride_h, const int stride_w,
        const int channel_per_deformable_group);


//#endif //DEFORM3D_NEW_PYTORCH_DEFORM_CONV3D_CUDA_H
<file_sep>import torch
from deform_conv3dl_modules import ConvOffset3d
from torch.autograd import Variable
import os
from gradcheck import gradcheck
from deform_conv3dl_functions import ConvOffset3dFunction

os.environ['CUDA_VISIBLE_DEVICES'] = '0'

batchsize = 2
c_in = 2
c_out = 3
inpu = 5
kernel = 3
stri = 2
pad = 1
out = int((inpu + 2 * pad - kernel) / stri + 1)
channel_per_group = 2
g_off = c_in // channel_per_group

conv_offset3d = ConvOffset3dFunction((stri, stri, stri), (pad, pad, pad), channel_per_group)

inputs = Variable(torch.rand(batchsize, c_in, inpu, inpu, inpu).type(torch.DoubleTensor).cuda(), requires_grad=True)
offsets = Variable(torch.rand(batchsize, g_off, out, out, out).type(torch.DoubleTensor).cuda(),
                   requires_grad=True)
weight = Variable(torch.rand(c_out, c_in, kernel, kernel, kernel).type(torch.DoubleTensor).cuda(),
                  requires_grad=True)

print(gradcheck(conv_offset3d, (inputs, offsets, weight)))
<file_sep>import deform_conv3dl_op as deform_conv
import torch
import os
from deform_conv3dl_functions import ConvOffset3dFunction

os.environ['CUDA_VISIBLE_DEVICES'] = '0'

batchsize = 1
c_in = 1
c_out = 1
inpu = 3
kernel = 1
stri = 1
pad = 0
out = int((inpu + 2 * pad - kernel) / stri + 1)
channel_per_group = 1
g_off = c_in // channel_per_group

# inpu = torch.cuda.DoubleTensor([[[[[1.0, 1, 1], [1.0, 2, 3], [1.0, 3, 1]]] * inpu] * c_in] * batchsize)
inpu = torch.cuda.DoubleTensor(
    [[[[[1.0] * inpu] * inpu, [[2.0] * inpu] * inpu, [[1.0] * inpu] * inpu]] * c_in] * batchsize)
offset = torch.cuda.DoubleTensor([[[[[2.] * out] * out] * out] * g_off] * batchsize)
weight = torch.cuda.DoubleTensor([[[[[1.0] * kernel] * kernel] * kernel] * c_in] * c_out)
tmp = offset.cpu().numpy()

padding = [pad, pad, pad]
stride = [stri, stri, stri]
conv_offset = ConvOffset3dFunction(stride, padding, channel_per_group)
output = conv_offset.forward(inpu, offset, weight)
tmp = output.cpu().numpy()

grad_output = torch.cuda.DoubleTensor([[[[[1.0] * out] * out] * out] * c_out] * batchsize)
columns = torch.zeros(weight.size(1) * weight.size(2) * weight.size(3) * weight.size(4),
                      output.size(2) * output.size(3) * output.size(4)).double().cuda()

# test for graph and grad
# inputs = Variable(torch.randn(batchsize,c_in,in_l,in_h,in_w)
#                   .double().cuda(),requires_grad=True)
# offsets = Variable(torch.randn(batchsize,g_c,out_l,out_h,out_w)
#                    .double().cuda(),requires_grad=True)
# weights = Variable(torch.randn(c_out,c_in,kernel_l,kernel_h,kernel_w)
#                    .double().cuda(), requires_grad=True)
# o1 = conv_offset(inputs, offsets, weights)
# o2 = Variable(grad_output,requires_grad=False)
# loss = (o1 - o2).sum()
# loss.backward()
# grad_i = inputs.grad.data.cpu().numpy()
# grad_o = offsets.grad.data.cpu().numpy()
# grad_w = weights.grad.data.cpu().numpy()
# ---------------------------------------------

grad_input = inpu.new(*inpu.size()).double().zero_()
grad_offset = offset.new(*offset.size()).double().zero_()
deform_conv.deform_conv_backward_input_offset_cuda(
    inpu, weight, offset, grad_output, columns, grad_input, grad_offset,
    pad, pad, pad,
    stri, stri, stri, channel_per_group)
grad_input_ = grad_input.cpu().numpy()
grad_offset_ = grad_offset.cpu().numpy()

grad_weight = weight.new(*weight.size()).double().zero_()
deform_conv.deform_conv_backward_weight_cuda(
    inpu, offset, grad_output, columns, grad_weight,
    pad, pad, pad,
    stri, stri, stri, channel_per_group)
grad_weight_ = grad_weight.cpu().numpy()

print('forward\n', tmp)
print('grad input\n', grad_input_)
# print('grad weight\n', grad_weight_)
print('grad offset\n', grad_offset_)
<file_sep>import math

import torch
import torch.nn as nn
from torch.nn.modules.module import Module
from torch.nn.modules.utils import _triple

from deform_conv3dl_functions import ConvOffset3dFunction


class ConvOffset3d(Module):
    def __init__(self,
                 in_channels,
                 out_channels,
                 kernel_size,
                 stride=1,
                 padding=0,
                 channel_per_group=1):
        super(ConvOffset3d, self).__init__()
        self.in_channels = in_channels
        self.out_channels = out_channels
        self.kernel_size = _triple(kernel_size)
        self.stride = _triple(stride)
        self.padding = _triple(padding)
        self.channel_per_group = channel_per_group

        self.weight = nn.Parameter(torch.cuda.DoubleTensor(out_channels, in_channels, *self.kernel_size))
        nn.init.kaiming_normal(self.weight.data, mode='fan_out')

    def forward(self, input, offset):
        return ConvOffset3dFunction(self.stride, self.padding, self.channel_per_group)(input, offset, self.weight)
<file_sep>import os.path as path
import torch
from torch.utils.ffi import create_extension

sources = ['lib/deform_conv3d_cuda.cpp']
headers = ['lib/deform_conv3d_cuda.h']
defines = [('WITH_CUDA', None)]
with_cuda = True

this_file = path.dirname(path.realpath(__file__))
print(this_file)
extra_objects = ['lib/deform_conv3d_cuda_kernel.cu.o']
extra_objects = [path.join(this_file, fname) for fname in extra_objects]

ffi = create_extension(
    'deform_conv3dl_op',
    headers=headers,
    sources=sources,
    define_macros=defines,
    relative_to=__file__,
    with_cuda=with_cuda,
    extra_objects=extra_objects
)

if __name__ == '__main__':
    assert torch.cuda.is_available(), 'Please install CUDA for GPU support.'
    ffi.build()
