Hey guys, I wrote an encoder-decoder with attention model to predict reverse strings - that is, when getting abc return cba.
I get good results for most of my runs, but every few runs, my network doesn’t converge at all.
Any suggestion how to check what may cause this?
my encoder and decoder:
class EncoderRNN(rnn.RNN):
	def __init__(self, hidden_size, emb_size, vocab_size, pre_trained_emb=None, n_layers=1, bidirect=True):
		super(EncoderRNN, self).__init__()
		self.n_layers = n_layers
		self.hidden_size = hidden_size
		self.bidirect = bidirect
		self.emb = self.create_emb(vocab_size, emb_size, pre_trained_emb)
		self.gru = nn.GRU(emb_size, hidden_size, batch_first=True, num_layers=n_layers, bidirectional=bidirect)
		self.opt = optim.Adam(self.params())

	def forward(self, input_sequence, hidden):
		embeddings = self.emb(input_sequence)
		output, hidden = self.gru(embeddings, hidden)
		return output, hidden

class DecoderRNN(rnn.RNN):
	def __init__(self, hidden_size, emb_size, vocab_size, pre_trained_emb=None):
		super(DecoderRNN, self).__init__()
		self.hidden_size = hidden_size
		self.n_layers = config.values.get('n_layers', 1)
		self.emb = self.create_emb(vocab_size, emb_size, pre_trained_emb)
		self.W1 = Par(hidden_size, hidden_size) # U_a in the paper
		self.W2 = Par(hidden_size, hidden_size) # W_a in the paper
		self.W3 = Par(emb_size+hidden_size, hidden_size)
		self.b2 = Par(hidden_size)
		self.b3 = Par(hidden_size)
		self.v = Par(hidden_size)

		self.gru = nn.GRU(hidden_size, hidden_size, num_layers=self.n_layers)
		self.linear = nn.Linear(hidden_size, vocab_size)
		self.opt = optim.Adam(self.params(), lr=0.01)

	def forward(self, prev_inp, hidden, enc_outputs):
		Uh = torch.matmul(enc_outputs, self.W1)
		Ws = torch.matmul(torch.cat(hidden, 1)[:,:self.hidden_size], self.W2)
		Wsb = torch.add(Ws, self.b2).unsqueeze(1)

		u = F.tanh(torch.add(Uh, Wsb))
		attn_weights = torch.mul(self.v, u).sum(2)
		attn_weights = F.softmax(attn_weights).unsqueeze(2)
		context_vector = torch.mul(attn_weights, enc_outputs).sum(1).squeeze(1)
		# s_i = f(s_i-1, y_i-1, c_i)
		prev_inp_emb = self.emb(prev_inp)
		res = torch.matmul(torch.cat([prev_inp_emb, context_vector], 1), self.W3)
		res = torch.add(res, self.b3).unsqueeze(0)
		res, hidden = self.gru(res, hidden.view(self.n_layers, -1, self.hidden_size))
		res = self.linear(res.squeeze(0))
		res = F.log_softmax(res)

		return res, hidden, attn_weights

A few functions that are shared for the encoder and decoder:
class RNN(nn.Module):
	def __init__(self):
		super(RNN, self).__init__()

	def initHidden(self, batch_size):
		return cuda(Variable(torch.zeros(self.n_layers * self.num_directions(), batch_size, self.hidden_size)))

	def create_emb(self, output_size, emb_size, pre_trained_emb_mat):
		emb = nn.Embedding(output_size, emb_size)
		if pre_trained_emb_mat is not None:
			emb.load_state_dict({'weight': pre_trained_emb_mat})
			for param in emb.parameters(): 
				param.requires_grad = False
		return emb

	def params(self):
		return (p for p in self.parameters() if p.requires_grad)

	def num_directions(self):
		return 2 if self.bidirect else 1	

And a few more functions
def cuda(var):
	if torch.cuda.is_available():
		return var.cuda()
	return var

def Arr(*sizes): return torch.randn(sizes)
def Par(*sizes): return torch.nn.Parameter(Arr(*sizes))

Where I initialized the encoder and decoder in the following way:
encoder = EncoderRNN(hidden_size, emb_size, vocab.n_words, n_layers=n_layers, bidirect=bidirect)
decoder = DecoderRNN(hidden_size*enc_num_of_directions, emb_size, vocab.n_words)

Notice the decoder state size is twice the size of the encoder.