In the end, adapting the code from this issue, I was able to do the following:
class OffsetConv2d(nn.Conv2d):
    """
    2d convolution with offset of central features
    
    See Also
    --------
    https://github.com/pytorch/pytorch/issues/47990
    """
    
    def __init__(self, in_channels, out_channels, kernel_size=3, stride=1, padding=0, bias=True):
        super().__init__(in_channels, out_channels, kernel_size, stride, padding, bias)
        self.k = kernel_size
        self.stride = stride
        self.padding = padding
        self.conv = nn.Parameter(torch.ones(out_channels, in_channels, kernel_size, kernel_size))
        
    def forward(self, x):
        k = self.k
        stride = self.stride
        h_in, w_in = x.shape[2], x.shape[3]

        padding = self.padding
        batch_size = x.shape[0]

        h_out = (h_in + 2 * padding - (k - 1) - 1) / stride + 1
        w_out = (w_in + 2 * padding - (k - 1) - 1) / stride + 1
        h_out, w_out = int(h_out), int(w_out)
        
        inp_unf = F.unfold(x, (k, k), padding=padding)
        
        
        # DO MY STUFF HERE
        # Offset each local block with its center
        
        out_unf = inp_unf.transpose(1, 2).matmul(self.conv.view(self.conv.size(0), -1).t()).transpose(1, 2)
        out = F.fold(out_unf, (h_out, w_out), (1, 1))
                        
        return out

I compared the above code with the traditional nn.Conv2d and I didn’t find any significant performance difference. So I will go ahead with this.
I hope that it is right and the above code will be useful to someone else in the same situation.