One of my friend solve this problem.
The solution is expanding index matrix and use gather
# for 3-dim input
X = torch.tensor([[[1, 2], [3, 4], [5, 6]], [[7, 8], [9, 10], [11, 12]]])
I = torch.tensor([[0, 2], [2, 1]])
eI = I[..., None].expand(-1, -1, X.size(2))    ## expanding index
Y = torch.gather(X, dim=1, index=eI).squeeze()

# for 4-dim input
X = torch.tensor([[[[0, 0], [1, 1]], [[2, 2], [3, 3]], [[4, 4], [5, 5]]], [[[6, 6], [7, 7]], [[8, 8], [9, 9]], [[10, 10], [11, 11]]]])
I = torch.tensor([[0, 2], [2, 1]])
eI = I[..., None, None].expand(-1, -1, X.size(2), X.size(3))    ## expanding index
Y = torch.gather(X, dim=1, index=eI).squeeze()