thanks for the reply,
my model is very simple:
class Net(nn.Module):
    def __init__(self, vocab_size):
        super(Net, self).__init__()
        self.vocab_size = vocab_size
        self.embed_size = 300
        self.hidden_size = 300
        self.linear = nn.Linear(1024, 300)
        self.dropout_rate = 0.5
        self.embedding = nn.Embedding(self.vocab_size, self.embed_size)
        self.dropout = nn.Dropout(self.dropout_rate)
        self.LSTM = nn.LSTM(self.embed_size, self.hidden_size, bidirectional=True)
        self.multimodal_linear = nn.Linear(600, 2)

    def forward(self, s, c):
        r = self.linear(c)
        self.LSTM.flatten_parameters()
        embedded = self.embedding(s)
        embedded = self.dropout(embedded)
        # Each batch has the same maxlen, how to make data loader with custom maxlen?
        input_lengths = [sent.shape[1]]* sent.shape[0]
        packed = torch.nn.utils.rnn.pack_padded_sequence(embedded, input_lengths, batch_first=True)
        output, hidden = self.LSTM(packed, None)
        output, _ = torch.nn.utils.rnn.pad_packed_sequence(output, batch_first=True)
        bi_text = hidden[0][0,:,:].squeeze()
        o = self.m_linear(torch.cat((bi_text, r), dim=1)) 
        return o

This is wrapped by another class, which has some other functions as well.
and in that class, say testNet
self.net = Net(..)
and I use testNet = nn.DataParallel(testNet)
and use testNet.module.parameters as optimizer parameters.