I am getting the same error in two ways:
> class EncoderCNN(nn.Module):
>     def __init__(self, embed_size, pre_trained_emb_size=2048):
>         """Load the pretrained ResNet Features"""
>         super(EncoderCNN, self).__init__()
>         self.linear = nn.Linear(pre_trained_emb_size, embed_size)
>         self.relu = nn.ReLU(embed_size)
>         self.bn = nn.BatchNorm1d(embed_size, momentum=0.01)
> 
>     def forward(self, images):
>         """Extracted feature vectors form input"""
>     
>         features = self.bn(self.relu(self.linear(images)))
>         return features
> 
> class TransformerEncoder(nn.Module):
>     def __init__(self, vocab_size, embed_size, d_model=300, nhead=6, dim_feedforward=512, dropout=0.1):
>         super(TransformerEncoder, self).__init__()
>         self.input_size = vocab_size
>         self.embed_size = embed_size
>         self.embedding = nn.Embedding(self.input_size, self.embed_size)
>         # Encoder CNN
>         self.encoder_cnn = EncoderCNN(embed_size=self.embed_size)
>         self.transformer_encoder_layer = nn.TransformerEncoderLayer(d_model, nhead, dim_feedforward)
> 
>     def forward(self, input, input_lengths, images):
>         input = input.reshape((input.shape[1], input.shape[0]))
>         embedded = self.embedding(input)
>         output = self.transformer_encoder_layer(embedded)
>         image_encodings = self.encoder_cnn(images)
>         return output, image_encodings

Even if I take the encoder cnn module out and wrap it in data parallel separately, I get the same error. the final returned tensors output, image_encodings have different batch sizes