I am using the following Class to implement Tiling as in Caffe
class Tiling(nn.Module):
    # Tiling layer from orginal caffe-VPGNet
    def __init__(self, x_dim, tile_dim):
        super(Tiling, self).__init__()
        self.b, d, self.h, self.w = x_dim
        self.tile_dim = tile_dim
        self.out_d = int(d / (tile_dim ** 2))  # 4
        out_h = int(self.h * tile_dim)  # 15
        out_w = int(self.w * tile_dim)  # 20
        self.tiled_out = (
            torch.FloatTensor(self.b, self.out_d, out_h, out_w)
            .fill_(0.0)
            .to(torch.device("cuda:0"))
        )

    def forward(self, x):
        #print("x ",x.requires_grad)
        for ds in range(self.out_d):
            d_start = ds * (self.tile_dim ** 2)
            d_end = (ds + 1) * (self.tile_dim ** 2)
            for hs in range(self.h):
                for ws in range(self.w):
                    tile_select = x[:, d_start:d_end, hs, ws]
                    #          tile_select = x[:, ds*(self.tile_dim**2):(ds+1)*(self.tile_dim**2), hs, ws]
                    out_tile = tile_select.view(self.b, self.tile_dim, self.tile_dim)
                    h_start = hs * self.tile_dim
                    h_end = (1 + hs) * self.tile_dim
                    w_start = ws * self.tile_dim
                    w_end = (1 + ws) * self.tile_dim
                    self.tiled_out[:, ds, h_start:h_end, w_start:w_end] = out_tile

        #print("tiled_out ",self.tiled_out.requires_grad)

        return self.tiled_out

While trying to train the model with this layer at the head, I get the following error
    total_loss.backward()
  File "/home/SandeepMenon/venv-e3d/lib/python3.6/site-packages/torch/tensor.py", line 221, in backward
    torch.autograd.backward(self, gradient, retain_graph, create_graph)
  File "/home/SandeepMenon/venv-e3d/lib/python3.6/site-packages/torch/autograd/__init__.py", line 132, in backward
    allow_unreachable=True)  # allow_unreachable flag
RuntimeError: Trying to backward through the graph a second time, but the saved intermediate results have already been freed. Specify retain_graph=True when calling backward the first time.

My operations in the Tiling layer should not be considered in the computation graph for backprop and should be considered as a custom Reshape layer.
I tried to detach() the tensor before assignment by changing the following line
                    tile_select = x[:, d_start:d_end, hs, ws]
to
                    tile_select = x[:, d_start:d_end, hs, ws].detach()
However, then I get the following error
>     total_loss.backward()
>   File "/home/SandeepMenon/venv-e3d/lib/python3.6/site-packages/torch/tensor.py", line 221, in backward
>     torch.autograd.backward(self, gradient, retain_graph, create_graph)
>   File "/home/SandeepMenon/venv-e3d/lib/python3.6/site-packages/torch/autograd/__init__.py", line 132, in backward
>     allow_unreachable=True)  # allow_unreachable flag
> RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn

How should I re-write my Tiling layer to resolve this.
Thank you