It doesn’t. Hum…here’s the piece of code I was trying that one…
def train_batch(data, encoder, decoder, optimizer, criterion):
    encoder.train()
    decoder.train()
    images, captions, lengths = data
    images = images.to(device)
    captions = captions.to(device)
    #images = image.to('cpu')
    #captions = captions.to('cpu')
    targets = pack_padded_sequence(captions, lengths.cpu(), batch_first=True)[0]
    #targets = pack_padded_sequence(captions, lengths.gpu(), batch_first=True)[0]
    features = encoder(images)
    outputs = decoder(features, captions, lengths)
    loss = criterion(outputs, targets)
    targets.detach.clone() #DEBUG
    features.detach.clone() #DEBUG
    decoder.zero_grad()
    encoder.zero_grad()
    loss.backward()
    optimizer.step()
    return loss