I’m also trying to do regression using BERT and getting an error about Longs? Not sure what I’m doing wrong here.
from transformers import DistilBertForSequenceClassification, AdamW, BertConfig

device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')

config = BertConfig()
config.num_labels = 1

model = BertForSequenceClassification(config).from_pretrained('bert-base-cased')
model.to(device)
model.train()

optim = AdamW(model.parameters(), lr=5e-5)

for epoch in range(3):
    for batch in dataloader:
        optim.zero_grad()
        input_ids = batch['input_ids'].to(device)
        print(input_ids)
        attention_mask = batch['attention_mask'].to(device)
        print(attention_mask)
        token_type_ids = batch['token_type_ids'].to(device)
        print(token_type_ids)
        labels = batch['labels'].to(device)
        print(labels)
        outputs = model(input_ids=input_ids, attention_mask=attention_mask, 
                        token_type_ids=token_type_ids, labels=labels)
        loss = outputs[0]
        loss.backward()
        optim.step()

model.eval()

Here’s the error (I printed out some data as well):
tensor([[ 101, 1198, 4841,  ...,    0,    0,    0],
        [ 101, 2960, 2254,  ...,    0,    0,    0],
        [ 101, 2866,  182,  ...,    0,    0,    0],
        ...,
        [ 101,  178,  112,  ...,    0,    0,    0],
        [ 101, 9294, 1128,  ...,    0,    0,    0],
        [ 101, 1268, 1185,  ...,    0,    0,    0]], device='cuda:0')
tensor([[1, 1, 1,  ..., 0, 0, 0],
        [1, 1, 1,  ..., 0, 0, 0],
        [1, 1, 1,  ..., 0, 0, 0],
        ...,
        [1, 1, 1,  ..., 0, 0, 0],
        [1, 1, 1,  ..., 0, 0, 0],
        [1, 1, 1,  ..., 0, 0, 0]], device='cuda:0')
tensor([[0, 0, 0,  ..., 0, 0, 0],
        [0, 0, 0,  ..., 0, 0, 0],
        [0, 0, 0,  ..., 0, 0, 0],
        ...,
        [0, 0, 0,  ..., 0, 0, 0],
        [0, 0, 0,  ..., 0, 0, 0],
        [0, 0, 0,  ..., 0, 0, 0]], device='cuda:0')
tensor([0.2135, 0.9005, 0.4206, 0.2755, 0.5373, 0.5537, 0.2492, 0.4841, 0.8241,
        0.3545, 0.2813, 0.5674, 0.4098, 0.5857, 0.9476, 0.6094, 0.2778, 0.2974,
        0.3362, 0.3490, 0.9035, 0.7904, 0.4856, 0.1117, 0.3851, 0.7932, 0.9066,
        0.3630, 0.2709, 0.8578, 0.2255, 0.3292], device='cuda:0')
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-66-2f7282efa3c9> in <module>()
     24         print(labels)
     25         outputs = model(input_ids=input_ids, attention_mask=attention_mask, 
---> 26                         token_type_ids=token_type_ids, labels=labels)
     27         loss = outputs[0]
     28         loss.backward()

5 frames
/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py in nll_loss(input, target, weight, size_average, ignore_index, reduce, reduction)
   2262                          .format(input.size(0), target.size(0)))
   2263     if dim == 2:
-> 2264         ret = torch._C._nn.nll_loss(input, target, weight, _Reduction.get_enum(reduction), ignore_index)
   2265     elif dim == 4:
   2266         ret = torch._C._nn.nll_loss2d(input, target, weight, _Reduction.get_enum(reduction), ignore_index)

RuntimeError: Expected object of scalar type Long but got scalar type Float for argument #2 'target' in call to _thnn_nll_loss_forward

Any help is appreciated!