@ptrblck
Thank you for the reply and apologies for my typo. I renamed x_detach to x.
My use case is to implement Tiling layer as in the caffe tiling layer mentioned in this repo
And yes I am getting the first error message by using x directly.
Code for former case
import torch
import torch.nn as nn

class Tiling(nn.Module):
    # Tiling layer from original caffe-VPGNet
    def __init__(self, x_dim, tile_dim):
        super(Tiling, self).__init__()
        self.b, d, self.h, self.w = x_dim
        self.tile_dim = tile_dim
        self.out_d = int(d / (tile_dim ** 2))  # 4
        self.out_h = int(self.h * tile_dim)  # 15
        self.out_w = int(self.w * tile_dim)  # 20
        self.tiled_out = (
            torch.FloatTensor(self.b, self.out_d, self.out_h, self.out_w)
            .fill_(0.0)
            .to(torch.device("cuda:0"))
        )

    def forward(self, x):
        # tiled_out = (
        #     torch.FloatTensor(self.b, self.out_d, self.out_h, self.out_w)
        #     .fill_(0.0)
        # ).to(torch.device("cuda:0"))

        for ds in range(self.out_d):
            d_start = ds * (self.tile_dim ** 2)
            d_end = (ds + 1) * (self.tile_dim ** 2)
            for hs in range(self.h):
                for ws in range(self.w):
                    tile_select = x[:, d_start:d_end, hs, ws]
                    out_tile = tile_select.view(self.b, self.tile_dim, self.tile_dim)
                    h_start = hs * self.tile_dim
                    h_end = (1 + hs) * self.tile_dim
                    w_start = ws * self.tile_dim
                    w_end = (1 + ws) * self.tile_dim
                    self.tiled_out[:, ds, h_start:h_end, w_start:w_end] = out_tile

        return self.tiled_out

class Test_Tiling(nn.Module):
    def __init__(self, batch_size=2):
        super().__init__()
        
        self.layer1 = nn.Conv2d(320, 320, kernel_size=1)
        tile_d = Tiling([batch_size, 320, 15, 20], 8)
        self.vpp = nn.Sequential(*[self.layer1, tile_d])

    def forward(self,x):
        out_1 = self.vpp(x)

        return out_1

if __name__ == "__main__":
    batch_size = 1
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

    model = Test_Tiling(batch_size=batch_size).to(device)
    loss_fn = nn.BCEWithLogitsLoss()
    optim_fn = torch.optim.SGD(model.parameters(), lr=0.0001, momentum=0.9)

    for epoch in range(5):
        inp = torch.FloatTensor(batch_size, 320, 480, 640).cuda()
        ground_truth = torch.FloatTensor(batch_size,5,120,160).cuda()
        y = model(inp)

        total_loss = loss_fn(y, ground_truth)

        # Zero optimizer
        optim_fn.zero_grad()

        # Backprop
        total_loss.backward()

        # Optimize weights
        optim_fn.step()

        print(f"Epoch:[{epoch}/{5}] loss:[{total_loss:.4f}] ")

Code for latter case (Change only in the Tiling class).
I added .detach() while reading from x and defining tiled_out inside forward function to not use the saved results as per the error message(not sure if this is doing it or not, but the first error goes away)
class Tiling(nn.Module):
    # Tiling layer from original caffe-VPGNet
    def __init__(self, x_dim, tile_dim):
        super(Tiling, self).__init__()
        self.b, d, self.h, self.w = x_dim
        self.tile_dim = tile_dim
        self.out_d = int(d / (tile_dim ** 2))  # 4
        self.out_h = int(self.h * tile_dim)  # 15
        self.out_w = int(self.w * tile_dim)  # 20
        # self.tiled_out = (
        #     torch.FloatTensor(self.b, self.out_d, self.out_h, self.out_w)
        #     .fill_(0.0)
        #     .to(torch.device("cuda:0"))
        # )

    def forward(self, x):
        tiled_out = (
            torch.FloatTensor(self.b, self.out_d, self.out_h, self.out_w)
            .fill_(0.0)
        ).to(torch.device("cuda:0"))

        for ds in range(self.out_d):
            d_start = ds * (self.tile_dim ** 2)
            d_end = (ds + 1) * (self.tile_dim ** 2)
            for hs in range(self.h):
                for ws in range(self.w):
                    tile_select = x[:, d_start:d_end, hs, ws].detach()
                    out_tile = tile_select.view(self.b, self.tile_dim, self.tile_dim)
                    h_start = hs * self.tile_dim
                    h_end = (1 + hs) * self.tile_dim
                    w_start = ws * self.tile_dim
                    w_end = (1 + ws) * self.tile_dim
                    tiled_out[:, ds, h_start:h_end, w_start:w_end] = out_tile

        return tiled_out