Check it before executing the forward and backward pass inside the training loop:
for data, target in loader:
    optimizer.zero_grad()
    print(model.my_rnn.training)
    output = model(input)
    loss = criterion(output, target)
    print(model.my_rnn.training)
    loss.backward() # this would most likely raise the error
    optimizer.step()