Ok, the problem is solved.
Using a reshape operation before index_select is more efficient.
# the example code
        pi = pi.reshape(-1, self.act_dim)
        obs = obs.reshape(N, -1)

        selected_agents_idx = []
        for batch_idx in range(N):
            batch_select_agent_idx = obs[batch_idx] + (batch_idx * self.map_size * self.map_size)
            selected_agents_idx.append(batch_select_agent_idx)
        selected_agents_idx = torch.cat(selected_agents_idx)
        
        action_selected = pi.index_select(0, selected_agents_idx)
        action_selected = action_selected.reshape(N, self.n_agents, -1)