If I understand your use case correctly, this should work:
N = 5
A = torch.randn(N, N)
x, y = torch.randint(0, N, (N,)), torch.randint(0, N, (N,))

B = torch.zeros(N, N)

# slow approach for verification
for i in range(N):
    for j in range(N):
        B[i, j] = A[x[i], y[j]]

# avoiding loops
C = A[x.unsqueeze(1), y]

# check for equal values
print((B == C).all())
> tensor(True)