I noticed that if I don’t save the hidden state in my rnn model, I don’t have any backward runtime error when calling loss.backward on the dqn network. But I need to save the hidden states in order for the rnn model to be useful.
This is how I defined my rnn model:
class LSTMState(nn.Module):
    def __init__(self, input_size, mini_batch_size):
        """
        :param input_size: Number of features in state
        This network has 1 output: reward.
        The LSTM cell outputs its representation of the state, which will be passed into DQN.
        """
        super(LSTMState, self).__init__()
        # initialize hidden state
        self.h = Variable(torch.zeros(mini_batch_size, input_size))
        # initialize cell state
        self.c = Variable(torch.zeros(mini_batch_size, input_size))
        self.hidden_dim = input_size

        # hidden size is number of features in the hidden state / size of output
        # input size is the number of features in incoming input
        self.lstm = nn.LSTMCell(input_size=input_size, hidden_size=input_size)
        self.fc = nn.Linear(input_size, 1) # the linear layer output represents predicted reward
        init.xavier_uniform(self.fc.weight)
        # no need for an activation function, since magnitude can be big, and rewards can be negative

    def forward(self, x, train=False):
        if train: # if training, we don't want to remember the hidden states?
            # initialize hidden state
            self.h = Variable(torch.zeros(x.size(0), self.hidden_dim))
            # initialize cell state
            self.c = Variable(torch.zeros(x.size(0), self.hidden_dim))
            h, c = self.lstm(x, (self.h, self.c))
            out = self.fc(self.h)
            return out, h
        else: # when testing, save the hidden and cell state
            self.h, self.c = self.lstm(x, (self.h, self.c))
            out = self.fc(self.h)
            # linear layer output out is the predicted reward, and self.h is the predicted state information
            return out, self.h

Never mind. The error arises if I save the hidden and cell states while training. But with this ^ definition it works, although it runs terribly.