@jekbradbury
So indeed, in order to avoid extra .contiguous() call, it’s strongly recommended to go with batch_first=False?
Cuz I think this function call would invoke memory copy to make contiguous array under the hood.
rnn = th.nn.LSTM(5, 10, 1, batch_first=True)  # batch_first

x = th.autograd.Variable(th.randn(1, 2, 5))  # batch size 1
out, state = rnn(x)
print(out.is_contiguous())    # TRUE

x = th.autograd.Variable(th.randn(2, 2, 5))  # batch size 2
out, state = rnn(x)
print(out.is_contiguous())    # FALSE

rnn = th.nn.LSTM(5, 10, 1)   # seq_len first

x = th.autograd.Variable(th.randn(2, 1, 5))  # batch size 1
out, state = rnn(x)
print(out.is_contiguous())    # TRUE

x = th.autograd.Variable(th.randn(2, 2, 5))  # batch size 2
out, state = rnn(x)
print(out.is_contiguous())    # TRUE