@glaringlee Thanks for your inputs again.
I guess I made it a  little bitte confused. Here are the complete signatures of all functions as well as the PYBIND11_MODULE. Hopefully I do not make any goofy  errors here.

> PYBIND11_MODULE

#include " roi_crop.h"

//namespace densecap {

PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
  m.def("BilinearSamplerBHWD_updateOutput", &BilinearSamplerBHWD_updateOutput, "BilinearSamplerBHWD_updateOutput");
  m.def("BilinearSamplerBHWD_updateGradInput", &BilinearSamplerBHWD_updateGradInput, "BilinearSamplerBHWD_updateGradInput");
}

//} // densecap



> Definition of interface: roi_crop.h

#ifndef ROI_CROP_CPU_H_
#define ROI_CROP_CPU_H_

#include <THC/THC.h>
#include <c10/core/Device.h>

#define USE_AT_TENSOR

#include "cpu/vision.h"

#ifdef WITH_CUDA
#include "cuda/vision.h"
#endif


#ifdef USE_AT_TENSOR
    int BilinearSamplerBHWD_updateOutput(const at::Tensor& inputImages, const at::Tensor& grids, at::Tensor& output) {
        int ret = 1;

        if (inputImages.type().is_cuda()) {
        #ifdef WITH_CUDA
            return BilinearSamplerBHWD_updateOutput_cuda(inputImages, grids, output);
        #else
            AT_ERROR("Not compiled with GPU support");
        #endif // WITH_CUDA
            }

          ret = BilinearSamplerBHWD_updateOutput_cpu(inputImages, grids, output);
          return ret;
    }

    int BilinearSamplerBHWD_updateGradInput(const at::Tensor& inputImages, const at::Tensor& grids, const at::Tensor& gradInputImages,
                                        const at::Tensor& gradGrids,  at::Tensor& gradOutput){
        int ret = 1;

        if (inputImages.type().is_cuda()) {
        #ifdef WITH_CUDA
            return BilinearSamplerBHWD_updateGradInput_cuda(inputImages, grids, gradInputImages, gradGrids, gradOutput);
        #else
            AT_ERROR("Not compiled with GPU support");
        #endif // WITH_CUDA
            }

          ret = BilinearSamplerBHWD_updateGradInput_cpu(inputImages, grids, gradInputImages, gradGrids, gradOutput);
      return ret;
    }
#else
    int BilinearSamplerBHWD_updateOutput(THCudaTensor* inputImages, THCudaTensor* grids, THCudaTensor* output) {
         printf("BilinearSamplerBHWD_updateOutput ");
            int ret = 1;

            if (inputImages->device_type() == c10::DeviceType::CUDA) {
        #ifdef WITH_CUDA
            return BilinearSamplerBHWD_updateOutput_cuda(inputImages, grids, output);
        #else
            AT_ERROR("Not compiled with GPU support");
        #endif // WITH_CUDA
            }

          ret = BilinearSamplerBHWD_updateOutput_cpu(inputImages, grids, output);
          return ret;
     }


    int BilinearSamplerBHWD_updateGradInput(THCudaTensor* inputImages, THCudaTensor* grids, THCudaTensor* gradInputImages,
                                        THCudaTensor* gradGrids,  THCudaTensor* gradOutput){
        int ret = 1;

        //if (inputImages.type().is_cuda()) {
        if (inputImages->device_type() == c10::DeviceType::CUDA) {
        #ifdef WITH_CUDA
            return BilinearSamplerBHWD_updateGradInput_cuda(inputImages, grids, gradInputImages, gradGrids, gradOutput);
        #else
            AT_ERROR("Not compiled with GPU support");
        #endif // WITH_CUDA
            }

          ret = BilinearSamplerBHWD_updateGradInput_cpu(inputImages, grids, gradInputImages, gradGrids, gradOutput);
      return ret;
    }
#endif  //USE_AT_TENSOR
#endif //#define ROI_CROP_CPU_H_


> Definition of interface for cpu <cpu/vision.h>

#pragma once
#include <torch/extension.h>
#define USE_AT_TENSOR

#ifdef USE_AT_TENSOR
    int BilinearSamplerBHWD_updateOutput_cpu(const at::Tensor& inputImages, const at::Tensor& grids, at::Tensor& output);
    int BilinearSamplerBHWD_updateGradInput_cpu(const at::Tensor& inputImages, const at::Tensor& grids, const at::Tensor& gradInputImages,
                                        const at::Tensor& gradGrids, at::Tensor& gradOutput);
#else
    int BilinearSamplerBHWD_updateOutput_cpu(THCudaTensor* inputImages, THCudaTensor* grids, THCudaTensor* output);
    int BilinearSamplerBHWD_updateGradInput_cpu(THCudaTensor* inputImages, THCudaTensor* grids, THCudaTensor* gradInputImages,
                                        THCudaTensor* gradGrids, THCudaTensor* gradOutput);
#endif  // USE_AT_TENSOR

The corresponding definition of interfaces for cuda
> Definition of interface for cuda <cuda/vision.h>
#pragma once
#include <torch/extension.h>
#define USE_AT_TENSOR

#ifdef USE_AT_TENSOR
    int BilinearSamplerBHWD_updateOutput_cuda(const at::Tensor& inputImages, const at::Tensor& grids, at::Tensor& output);
    int BilinearSamplerBHWD_updateGradInput_cuda(const at::Tensor& inputImages, const at::Tensor& grids, const at::Tensor& gradInputImages,
                                        const at::Tensor& gradGrids, at::Tensor& gradOutput);
#else
    int BilinearSamplerBHWD_updateOutput_cuda(THCudaTensor* inputImages, THCudaTensor* grids, THCudaTensor* output);
    int BilinearSamplerBHWD_updateGradInput_cuda(THCudaTensor* inputImages, THCudaTensor* grids, THCudaTensor* gradInputImages,
                                        THCudaTensor* gradGrids, THCudaTensor* gradOutput);
#endif // USE_AT_TENSOR

In general, BilinearSamplerBHWD_updateOutput is the interface for python codes.
At the running time, depending on the device info, the functions are directed to call BilinearSamplerBHWD_updateOutput_cpu or BilinearSamplerBHWD_updateOutput_cuda.
Here is a brief schematic representation of this calling sequence.
Python Module --> BilinearSamplerBHWD_updateOutput 
                        | -->BilinearSamplerBHWD_updateOutput_cpu
                        |-->BilinearSamplerBHWD_updateOutput_cuda  

The definition of Python Module is given as follows:
class RoICropFunction(Function):
    @staticmethod
    def forward(self, input1, input2):
        self.input1 = input1.clone()
        self.input2 = input2.clone()
        output = input2.new(input2.size()[0], input1.size()[1], input2.size()[1], input2.size()[2]).zero_()
        assert output.get_device() == input1.get_device(), "output and input1 must on the same device"
        assert output.get_device() == input2.get_device(), "output and input2 must on the same device"
        # I'll freeze here if C++ interface is declared with THCudaTensor
        _C.BilinearSamplerBHWD_updateOutput(input1, input2, output)
        self.save_for_backward(output)
        return output

    @staticmethod
    def backward(ctx, grad_output):
        #result, = ctx.saved_tensors
        grad_input1 = ctx.input1.new(ctx.input1.size()).zero_()
        grad_input2 = ctx.input2.new(ctx.input2.size()).zero_()
        _C.BilinearSamplerBHWD_updateGradInput(ctx.input1, ctx.input2, grad_input1, grad_input2,
                                                          grad_output)
        return grad_input1, grad_input2

Hopefully, this helps us to locate the bugs. Thanks.