ok so I think I got it. I guess the inputs themselves need to be reversed when passing them through the backward/ reverse direction. Here is the code:
with torch.no_grad():
    for i in range(hx.shape[0]):
        print(hx[i])
print("--------------------------------")
print("--------------------------------")

tanh = nn.Tanh()
output = []
hid_forward = torch.zeros(1, 5, dtype = torch.float32)
hid_reverse = torch.zeros(1, 5, dtype = torch.float32)

# Forward
for i in range(input.shape[0]):
    x = input[i]
    
    i_forward = x @ torch.transpose(wih, 0, 1) + bih
    h_forward = hid_forward @ torch.transpose(whh, 0, 1) + bhh
    hid_forward = tanh(i_forward + h_forward)
    

# Reverse
reverse_input = torch.flip(input, (0, 1))
    
for i in range(reverse_input.shape[0]):
    x = reverse_input[i]
    
    i_reverse = x @ torch.transpose(wihr, 0, 1) + bihr
    h_reverse = hid_reverse @ torch.transpose(whhr, 0, 1) + bhhr
    hid_reverse = tanh(i_reverse + h_reverse)
    
print(hid_forward)
print(hid_reverse)
print("-----------")