Here is what I believe to be the best way to do it:
import torch
import torch.nn as nn
from torch import _VF as F


class RNN(nn.Module):
	def __init__(self, input_size, hidden_size):
		super(RNN, self).__init__()
		self.hidden_size = hidden_size
		self.hh_w = torch.rand([hidden_size, hidden_size])
		self.hi_w = torch.rand([hidden_size, input_size])
		self.no_fast_weights = hidden_size * hidden_size + hidden_size * input_size

	def update_weights(self, update, idx, update_func):
		end_of_weight_idx = idx + self.no_fast_weights
		end_of_hh_idx = idx + self.hidden_size * self.hidden_size
		hh_update = update[idx:end_of_hh_idx, :].reshape(self.hh_w.shape)
		hi_update = update[end_of_hh_idx:end_of_weight_idx, :].reshape(self.hi_w.shape)
		self.hh_w = update_func(self.hh_w, hh_update)
		self.hi_w = update_func(self.hi_w, hi_update)


	def forward(self, x):
		# F.rnn_tanh(torch.rand([3,3]), (self.input_w, self.hidden_w), False, 1, 0.0, True, False, False)
		hx = torch.zeros(1, x.shape[0], self.hidden_size)
		out, hid = F.rnn_tanh(x, hx, [self.hi_w, self.hh_w], False, 1, 0.0, True, False, False)
		return out, hid

which is based off a hypernetwork implementation (which is more straightforward since they are predicting the weights rather than weight updates, and using a built-in F module, while this is using some of the internals of pytorch. GitHub - g1910/HyperNetworks: PyTorch implementation of HyperNetworks (Ha et al., ICLR 2017) for R 1)