hello, when i use the  pack sequence -> recurrent network -> unpack sequence  pattern in a LSTM training with nn.DataParallel, i encounter a very strange problem.
here is my code:
class LSTM(nn.Module):
    def __init__(self, input_size, hidden_layer, hidden_size, num_classes, 
            rnn_type='lstm', dropout=0.0, bidirect=True, residual=False):
        super(LSTM, self).__init__()
        if bidirect:
            layer = [nn.LSTM( input_size=input_size, hidden_size=hidden_size, batch_first=True, bidirectional=bidirect, dropout=dropout)] 
            for i in range(hidden_layer):
                if i == hidden_layer-1:
                    layer.append(nn.LSTM(hidden_size*2, num_classes, batch_first=True, dropout=0.0, bidirectional=bidirect))
                else:
                    layer.append(nn.LSTM(hidden_size*2, hidden_size, batch_first=True, dropout=0.0, bidirectional=bidirect))
            self.lstm = nn.Sequential(*layer)
        else:
            pass
   def forward (self, x, input_len):
        
        _, idx_sort = th.sort(input_len, dim=0, descending=True)
        _, idx_unsort = th.sort(idx_sort, dim=0)
        x = x.index_select(0, Variable(idx_sort))
        input_len = input_len[idx_sort]
        x = x.cuda()
        max_length = x.shape[1]
        x = rnn_pack.pack_padded_sequence(x, input_len, batch_first=True)
        _x = self.lstm(x)_
        out , _ = rnn_pack.pad_packed_sequence(x, total_length=max_length, batch_first=True)
        out = out.index_select(0, Variable(idx_unsort))
        return out

def train_one_epoch(nnet, criterion, optimizer, train_loader, num_parallel, is_rnn=True):
    nnet.train()
    for index, (key, feats, labels, len_list) in enumerate(train_loader):
        labels = labels.view(labels.shape[0], labels.shape[1])
        input_len = np.array(len_list)
        optimizer.zero_grad()
        if is_rnn:
            label_mat = labels.view(labels.size(0) * labels.size(1))
        targets = Variable(label_mat.cuda())
        input_len = th.from_numpy(input_len)
        out = nnet(feats, input_len)
         ……………………

nnet = LSTM((args.left_context + args.right_context + 1) * args.feat_dim, args.hidden_layer, args.hidden_size, args.num_classes, rnn_type=net_type, dropout=args.dropout, bidirect=bidirect, residual=residual)
nnet = nn.DataParallel(nnet, device_ids=[0,1,2])

when I run the main script, the error occurs as follows:
Traceback (most recent call last):
  File "./train/train_rnn_pack_sort.py", line 195, in <module>
    train(args)
  File "./train/train_rnn_pack_sort.py", line 150, in train
    tr_frame_acc = train_epoch(nnet, criterion, optimizer, train_loader, num_parallel, train_dataset.num_frames, is_rnn=True)
  File "./train/train_rnn_pack_sort.py", line 60, in train_epoch
    train_frame, pos_frames = common_pack_sort.train_one_epoch(nnet, criterion, optimizer, train_loader, num_parallel, is_rnn)
  File "/search/speech/wangqingnan/asr_tools/pytorch/asr_egs/common/common_pack_sort.py", line 107, in train_one_epoch
    out = nnet(feats, input_len)
  File "/search/speech/wangqingnan/Anaconda/anaconda2/lib/python2.7/site-packages/torch/nn/modules/module.py", line 491, in __call__
    result = self.forward(*input, **kwargs)
  File "/search/speech/wangqingnan/Anaconda/anaconda2/lib/python2.7/site-packages/torch/nn/parallel/data_parallel.py", line 114, in forward
    outputs = self.parallel_apply(replicas, inputs, kwargs)
  File "/search/speech/wangqingnan/Anaconda/anaconda2/lib/python2.7/site-packages/torch/nn/parallel/data_parallel.py", line 124, in parallel_apply
    return parallel_apply(replicas, inputs, kwargs, self.device_ids[:len(replicas)])
  File "/search/speech/wangqingnan/Anaconda/anaconda2/lib/python2.7/site-packages/torch/nn/parallel/parallel_apply.py", line 65, in parallel_apply
    raise output
AttributeError: 'tuple' object has no attribute 'size'

and if i use “x = self.lstm(x.data)” instead of “x = self.lstm(x)” in the forward method,the error shows as follows:
Traceback (most recent call last):
  File "./train/train_rnn_pack_sort.py", line 195, in <module>
    train(args)
  File "./train/train_rnn_pack_sort.py", line 150, in train
    tr_frame_acc = train_epoch(nnet, criterion, optimizer, train_loader, num_parallel, train_dataset.num_frames, is_rnn=True)
  File "./train/train_rnn_pack_sort.py", line 60, in train_epoch
    train_frame, pos_frames = common_pack_sort.train_one_epoch(nnet, criterion, optimizer, train_loader, num_parallel, is_rnn)
  File "/search/speech/wangqingnan/asr_tools/pytorch/asr_egs/common/common_pack_sort.py", line 107, in train_one_epoch
    out = nnet(feats, input_len)
  File "/search/speech/wangqingnan/Anaconda/anaconda2/lib/python2.7/site-packages/torch/nn/modules/module.py", line 491, in __call__
    result = self.forward(*input, **kwargs)
  File "/search/speech/wangqingnan/Anaconda/anaconda2/lib/python2.7/site-packages/torch/nn/parallel/data_parallel.py", line 114, in forward
    outputs = self.parallel_apply(replicas, inputs, kwargs)
  File "/search/speech/wangqingnan/Anaconda/anaconda2/lib/python2.7/site-packages/torch/nn/parallel/data_parallel.py", line 124, in parallel_apply
    return parallel_apply(replicas, inputs, kwargs, self.device_ids[:len(replicas)])
  File "/search/speech/wangqingnan/Anaconda/anaconda2/lib/python2.7/site-packages/torch/nn/parallel/parallel_apply.py", line 65, in parallel_apply
    raise output
RuntimeError: input must have 3 dimensions, got 2

However, I remember that with one GPU, “x” is still a PackedSequence object, and there is no problems in the forward propagation.
Any help would be appreciated…