GPT2 eval with attention_mask not returning expected result
is this not correct use of attention_mask with padding (trying to batch sentence eval) or is it a bug? Shouldn't the return result be the same in each case because of the mask?

import torch
from transformers import GPT2LMHeadModel
model = GPT2LMHeadModel.from_pretrained('gpt2', return_dict=True)

```
with torch.no_grad():
    model.eval()
    loss,logits = model(input_ids=torch.tensor([15496,11,616]), attention_mask=torch.tensor([1,1,1]), labels=torch.tensor([15496,11,616]))[:2]
    print(loss)  # 3.0698 correct answer
    loss,logits = model(input_ids=torch.tensor([50256,15496,11,616]), attention_mask=torch.tensor([0,1,1,1]), labels=torch.tensor([50256,15496,11,616]))[:2]
    print(loss)  # 8.96 - shouldn't it be 3.0698 as above?
```
Hey @dan-i, 

1) You should mask the labels of the first token it's not enough to set the attention_mask to 0 -> `labels=torch.tensor([-100, ...])`
2) You have to change the `position_ids` to ensure the expected behavior in GPT2, *i.e.* make sure that in the second case the position_ids are [0, 0, 1, 2] instead of [0, 1, 2, 3]<|||||>thank patrick.  seems to work if you just do labels to -100 where the pads are without mask and position. at least it returns same result as individual sentence scores. for anybody else that needs gpt2 batch sentence perplexity scores.

```
def calc_loss(sentences,inputs,logits):
    # calc loss from logits returned from model
    lines_len = torch.sum(inputs['attention_mask'], dim=1)                  # quick way to get sent len. mask isn't used for anything else
    for line_ind in range(len(sentences)):                                  
        log_prob = 0.0
        for token_ind in range(lines_len[line_ind] - 1):                    
            token_prob = F.softmax(logits[line_ind, token_ind], dim=0)
            token_id = inputs['input_ids'][line_ind, token_ind + 1]
            log_prob += torch.log(token_prob[token_id])
        sentloss=abs(log_prob)
        toks=lines_len[line_ind]
        loss=sentloss/(toks-1)
        print(sentences[line_ind],' loss=',loss.item(),'toks=',toks.item(),'sentloss=',sentloss.item())

sentences=["Hello, my","Hello, my dog","Hello, my dog is a dog"]
tokenizer.pad_token = tokenizer.eos_token # set pad to eos
inputs = tokenizer(sentences, return_tensors="pt", padding=True)  # right side padding as usual
# mask of label_id's
label_ids=inputs['input_ids'].clone()
label_ids[label_ids==tokenizer.encode(tokenizer.pad_token)[0]] = -100
# one shot score the batch
with torch.no_grad():
    model.eval()
    loss, logits  = model(input_ids=inputs['input_ids'], labels=label_ids)[:2]
# calc loss per sentence
calc_loss(sentences,inputs,logits)
```
outputs:
Hello, my  loss= 3.069786548614502 toks= 3 sentloss= 6.139573097229004
Hello, my dog  loss= 4.247798442840576 toks= 4 sentloss= 12.74339485168457
Hello, my dog is a dog  loss= 3.671175003051758 toks= 7 sentloss= 22.027050018310547
