<p dir="auto">...................................................................................................</p>
<p dir="auto">Sorry, I have to reopen this because I think this is a bug within the RNN module and I could't get help from discuss.pytorch</p>
<p dir="auto">I run a similar experiment with one linear layer neural network and the output is correct. So a reasonable explanation would be the RNN module keep different set of parameters (for optimization reason?) and didn't mix up at the end of the batch training. The following code will duplicate this problem.</p>
<p dir="auto">........................................................................................................</p>
<p dir="auto">I encounter this problem when I am trying to implement seq2seq to familiarize with this new framework. This issue seems related to parameters sharing in mini-batch. I setup a dummy training set with only one mini-batch. This mini-batch has 3 data entries in it. All with the same input, and different outputs:</p>
<p dir="auto">training_data = [<br>
[[[4,5,1,7,8],[4,5,1,7,8],[4,5,1,7,8]], (input 1)<br>
[[0],[0],[0]], (input 2)<br>
[[1],[3],[5]]] (target)<br>
]</p>
<p dir="auto">In theory, the model will never learn this data because the contradiction. However, the loss reach near 0 after only a few hundred epochs. But if I split 3 data entries into 3 mini-batch, the model will not learn the data set which should be the correct result.</p>
<p dir="auto">So the model must be keeping different set of parameter for each position in mini-batch? And the parameters are not updated to be the same after each mini-batch forward-backward? Can someone tell me if I misunderstood something?</p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="import torch
import torch.autograd as autograd
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim



training_data = [
	[[[4,5,1,7,8],[4,5,1,7,8],[4,5,1,7,8]], [[0],[0],[0]], [[1],[3],[5]]]
	]


def prepare_sequence(all_seq):
	return autograd.Variable(torch.LongTensor(all_seq)).cuda()

class Seq2seqLSTM(nn.Module):

	def __init__(self, vocab_size, target_size, embed_dim, hidden_dim, num_layers):

		super(Seq2seqLSTM, self).__init__()
		self.hidden_dim = hidden_dim
		self.embed_dim = embed_dim
		self.vocab_size = vocab_size
		self.target_size = target_size
		self.num_layers = num_layers

		self.word_embeddings = nn.Embedding(vocab_size, embed_dim)
		self.encoder = nn.LSTM(embed_dim, hidden_dim, num_layers)
		self.decoder = nn.LSTM(embed_dim, hidden_dim, num_layers)

		self.curr_hidden = None
		self.hidden2tag = nn.Linear(hidden_dim, target_size)

	def init_hidden(self, batch_size):
		return (autograd.Variable(torch.zeros(self.num_layers, batch_size, self.hidden_dim)).cuda(),
			autograd.Variable(torch.zeros(self.num_layers, batch_size, self.hidden_dim)).cuda())

	def forward(self, enc_seq, dec_seq):

		batch_size = enc_seq.size()[0]
		self.curr_hidden = self.init_hidden(batch_size)

		enc_embeds = self.word_embeddings(enc_seq)
		dec_embeds = self.word_embeddings(dec_seq)

		enc_out, self.curr_hidden = self.encoder(
			enc_embeds.view(-1, batch_size, self.embed_dim), self.curr_hidden)
		dec_out, self.curr_hidden = self.decoder(
			dec_embeds.view(-1, batch_size, self.embed_dim), self.curr_hidden)

		tag_space = self.hidden2tag(dec_out.view(batch_size * len(dec_out), -1))
		tag_scores = F.log_softmax(tag_space)
		return tag_scores

EMBED_DIM = 10
HIDDEN_DIM = 10
VOCAB_SIZE = 10
TARGET_SIZE = 10
NUM_LAYERS = 2

model = Seq2seqLSTM(VOCAB_SIZE, TARGET_SIZE, EMBED_DIM, HIDDEN_DIM, NUM_LAYERS).cuda()
loss_function = nn.NLLLoss()
optimizer = optim.Adam(model.parameters())
print model

for epoch in range(1000):

	one_batch = map(lambda x:prepare_sequence(x), training_data[epoch%len(training_data)])
	enc_inp = one_batch[0]
	dec_inp = one_batch[1]
	target = one_batch[2]

	model.zero_grad()
	tag_scores = model(enc_inp, dec_inp)

	loss = loss_function(tag_scores, target.view(-1))
	loss.backward()
	optimizer.step()

	print loss"><pre class="notranslate"><code class="notranslate">import torch
import torch.autograd as autograd
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim



training_data = [
	[[[4,5,1,7,8],[4,5,1,7,8],[4,5,1,7,8]], [[0],[0],[0]], [[1],[3],[5]]]
	]


def prepare_sequence(all_seq):
	return autograd.Variable(torch.LongTensor(all_seq)).cuda()

class Seq2seqLSTM(nn.Module):

	def __init__(self, vocab_size, target_size, embed_dim, hidden_dim, num_layers):

		super(Seq2seqLSTM, self).__init__()
		self.hidden_dim = hidden_dim
		self.embed_dim = embed_dim
		self.vocab_size = vocab_size
		self.target_size = target_size
		self.num_layers = num_layers

		self.word_embeddings = nn.Embedding(vocab_size, embed_dim)
		self.encoder = nn.LSTM(embed_dim, hidden_dim, num_layers)
		self.decoder = nn.LSTM(embed_dim, hidden_dim, num_layers)

		self.curr_hidden = None
		self.hidden2tag = nn.Linear(hidden_dim, target_size)

	def init_hidden(self, batch_size):
		return (autograd.Variable(torch.zeros(self.num_layers, batch_size, self.hidden_dim)).cuda(),
			autograd.Variable(torch.zeros(self.num_layers, batch_size, self.hidden_dim)).cuda())

	def forward(self, enc_seq, dec_seq):

		batch_size = enc_seq.size()[0]
		self.curr_hidden = self.init_hidden(batch_size)

		enc_embeds = self.word_embeddings(enc_seq)
		dec_embeds = self.word_embeddings(dec_seq)

		enc_out, self.curr_hidden = self.encoder(
			enc_embeds.view(-1, batch_size, self.embed_dim), self.curr_hidden)
		dec_out, self.curr_hidden = self.decoder(
			dec_embeds.view(-1, batch_size, self.embed_dim), self.curr_hidden)

		tag_space = self.hidden2tag(dec_out.view(batch_size * len(dec_out), -1))
		tag_scores = F.log_softmax(tag_space)
		return tag_scores

EMBED_DIM = 10
HIDDEN_DIM = 10
VOCAB_SIZE = 10
TARGET_SIZE = 10
NUM_LAYERS = 2

model = Seq2seqLSTM(VOCAB_SIZE, TARGET_SIZE, EMBED_DIM, HIDDEN_DIM, NUM_LAYERS).cuda()
loss_function = nn.NLLLoss()
optimizer = optim.Adam(model.parameters())
print model

for epoch in range(1000):

	one_batch = map(lambda x:prepare_sequence(x), training_data[epoch%len(training_data)])
	enc_inp = one_batch[0]
	dec_inp = one_batch[1]
	target = one_batch[2]

	model.zero_grad()
	tag_scores = model(enc_inp, dec_inp)

	loss = loss_function(tag_scores, target.view(-1))
	loss.backward()
	optimizer.step()

	print loss
</code></pre></div>
<p dir="auto">This happens when the index tensor contains duplicate elements. If this is not allowed index_copy_ should raise an exception when this happens and we should fix the test. Otherwise we need to fix the autograd op.</p>
<p dir="auto">Here's an example:</p>
<div class="highlight highlight-source-python notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="import torch
from torch.autograd import *
x = Variable(torch.zeros(5, 5), requires_grad=True)
print(x)
y = Variable(torch.range(1, 25).view(5, 5), requires_grad=True)
print(y)
idx = Variable(torch.LongTensor([0, 0, 0, 0, 0]))
z = x.index_copy(0, idx, y)
print(z)  # Note only the last row of y is copied to the first row of z. No other rows of y are used.
z.backward(torch.ones(5, 5))
print(y.grad)  # Incorrectly all ones. Only the last row of y.grad should be non-zero."><pre class="notranslate"><span class="pl-k">import</span> <span class="pl-s1">torch</span>
<span class="pl-k">from</span> <span class="pl-s1">torch</span>.<span class="pl-s1">autograd</span> <span class="pl-k">import</span> <span class="pl-c1">*</span>
<span class="pl-s1">x</span> <span class="pl-c1">=</span> <span class="pl-v">Variable</span>(<span class="pl-s1">torch</span>.<span class="pl-en">zeros</span>(<span class="pl-c1">5</span>, <span class="pl-c1">5</span>), <span class="pl-s1">requires_grad</span><span class="pl-c1">=</span><span class="pl-c1">True</span>)
<span class="pl-en">print</span>(<span class="pl-s1">x</span>)
<span class="pl-s1">y</span> <span class="pl-c1">=</span> <span class="pl-v">Variable</span>(<span class="pl-s1">torch</span>.<span class="pl-en">range</span>(<span class="pl-c1">1</span>, <span class="pl-c1">25</span>).<span class="pl-en">view</span>(<span class="pl-c1">5</span>, <span class="pl-c1">5</span>), <span class="pl-s1">requires_grad</span><span class="pl-c1">=</span><span class="pl-c1">True</span>)
<span class="pl-en">print</span>(<span class="pl-s1">y</span>)
<span class="pl-s1">idx</span> <span class="pl-c1">=</span> <span class="pl-v">Variable</span>(<span class="pl-s1">torch</span>.<span class="pl-v">LongTensor</span>([<span class="pl-c1">0</span>, <span class="pl-c1">0</span>, <span class="pl-c1">0</span>, <span class="pl-c1">0</span>, <span class="pl-c1">0</span>]))
<span class="pl-s1">z</span> <span class="pl-c1">=</span> <span class="pl-s1">x</span>.<span class="pl-en">index_copy</span>(<span class="pl-c1">0</span>, <span class="pl-s1">idx</span>, <span class="pl-s1">y</span>)
<span class="pl-en">print</span>(<span class="pl-s1">z</span>)  <span class="pl-c"># Note only the last row of y is copied to the first row of z. No other rows of y are used.</span>
<span class="pl-s1">z</span>.<span class="pl-en">backward</span>(<span class="pl-s1">torch</span>.<span class="pl-en">ones</span>(<span class="pl-c1">5</span>, <span class="pl-c1">5</span>))
<span class="pl-en">print</span>(<span class="pl-s1">y</span>.<span class="pl-s1">grad</span>)  <span class="pl-c"># Incorrectly all ones. Only the last row of y.grad should be non-zero.</span></pre></div>
<p dir="auto">cc <a class="user-mention notranslate" data-hovercard-type="user" data-hovercard-url="/users/ezyang/hovercard" data-octo-click="hovercard-link-click" data-octo-dimensions="link_type:self" href="https://github.com/ezyang">@ezyang</a> <a class="user-mention notranslate" data-hovercard-type="user" data-hovercard-url="/users/albanD/hovercard" data-octo-click="hovercard-link-click" data-octo-dimensions="link_type:self" href="https://github.com/albanD">@albanD</a> <a class="user-mention notranslate" data-hovercard-type="user" data-hovercard-url="/users/zou3519/hovercard" data-octo-click="hovercard-link-click" data-octo-dimensions="link_type:self" href="https://github.com/zou3519">@zou3519</a> <a class="user-mention notranslate" data-hovercard-type="user" data-hovercard-url="/users/gqchen/hovercard" data-octo-click="hovercard-link-click" data-octo-dimensions="link_type:self" href="https://github.com/gqchen">@gqchen</a> <a class="user-mention notranslate" data-hovercard-type="user" data-hovercard-url="/users/pearu/hovercard" data-octo-click="hovercard-link-click" data-octo-dimensions="link_type:self" href="https://github.com/pearu">@pearu</a> <a class="user-mention notranslate" data-hovercard-type="user" data-hovercard-url="/users/nikitaved/hovercard" data-octo-click="hovercard-link-click" data-octo-dimensions="link_type:self" href="https://github.com/nikitaved">@nikitaved</a> <a class="user-mention notranslate" data-hovercard-type="user" data-hovercard-url="/users/soulitzer/hovercard" data-octo-click="hovercard-link-click" data-octo-dimensions="link_type:self" href="https://github.com/soulitzer">@soulitzer</a> <a class="user-mention notranslate" data-hovercard-type="user" data-hovercard-url="/users/lezcano/hovercard" data-octo-click="hovercard-link-click" data-octo-dimensions="link_type:self" href="https://github.com/lezcano">@lezcano</a> <a class="user-mention notranslate" data-hovercard-type="user" data-hovercard-url="/users/Varal7/hovercard" data-octo-click="hovercard-link-click" data-octo-dimensions="link_type:self" href="https://github.com/Varal7">@Varal7</a> <a class="user-mention notranslate" data-hovercard-type="user" data-hovercard-url="/users/ssnl/hovercard" data-octo-click="hovercard-link-click" data-octo-dimensions="link_type:self" href="https://github.com/ssnl">@ssnl</a></p>