How to compute loss with HuggingFace transformers?
Hello,

Is it possible to train HuggingFace TransfoXLLMHeadModel on a dataset different than WikiText103, say, on the combined WikiText2 and WikiText103 dataset?

Below are my code:
```js

# Import packages 
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import TransfoXLConfig, TransfoXLTokenizer, TransfoXLLMHeadModel
from transformers import AdamW, WarmupLinearSchedule
import spacy
import torchtext
from torchtext.data.utils import get_tokenizer
from torchtext.data import Field, BPTTIterator, TabularDataset 
import tensorflow as tf
import math
import random
import numpy as np
import pandas as pd 
import time

# set hyperparameters for this experiment
bptt = 30
batch_size = 64
lr = 0.01 # learning rate
criterion = nn.CrossEntropyLoss() # loss criterion

# define tokenizer
en = spacy.load('en')

def Sp_Tokenizer(text): 
    return [tok.text for tok in en.tokenizer(text)]

# define the English text field
TEXT = Field(tokenize = Sp_Tokenizer,
             init_token='< sos >',
             eos_token='< eos >',
             unk_token='< unk >',
             tokenizer_language='en',
             lower=True)

# load the datasets
train_Wiki2, val_Wiki2, test_Wiki2 = torchtext.datasets.WikiText2.splits(TEXT)
train_Wiki103, val_Wiki103, test_Wiki103 = torchtext.datasets.WikiText103.splits(TEXT)

# Define device
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

# build vocabulary based on the defined field and
# the combined WikiText2 and WikiText103 datasets.
TEXT.build_vocab(train_Wiki2, val_Wiki2, test_Wiki2, 
                 train_Wiki103, val_Wiki103, test_Wiki103)

# set hyperparameter ntokens
ntokens = len(TEXT.vocab.stoi)

## specify the transformer-XL model that we are going to use.
#
# define transformer-XL configuration.
transfoXLconfig = TransfoXLConfig(vocab_size_or_config_json_file = ntokens,
                                  cutoffs = [20000, 40000, 200000], 
                                  d_model = 64, 
                                  d_embed = 64, 
                                  n_head = 16, 
                                  d_head = 64,
                                  n_layer = 5,
                                  attn_type = 0,
                                  dropout = 0.1, 
                                  output_hidden_states = True,
                                  output_attentions = True)

# define the transformer-XL model based on the specified configuration.
model = TransfoXLLMHeadModel(transfoXLconfig)

# add new tokens to the embeddings of our model
model.resize_token_embeddings(ntokens)

# define BPTTiterators
#
train_iter, val_iter, test_iter = BPTTIterator.splits(
            (train_Wiki2, val_Wiki2, test_Wiki2),
            batch_size = batch_size,
            bptt_len= bptt,
            sort_key=lambda x: len(x.text),
            sort_within_batch = True,
            shuffle = False,
            device= device,
            repeat=False)

train = next(iter(train_iter))
val = next(iter(train_iter))
test = next(iter(test_iter))

```

and now I am trying to write the train function but I am not sure how exactly I should proceed.
Below is what I tried:

```js

# define the hyperparameters for running the train function.
train = train 
optimizer = AdamW(model.parameters())
scheduler = WarmupLinearSchedule(optimizer = optimizer, 
                                 warmup_steps = 200, 
                                 t_total = 1000,
                                 last_epoch = -1)
model.train()

# define the train function
def train(model, train, bptt, criterion, optimizer, scheduler, ntokens, log_interval):

    # initialize total_loss to 0
    total_loss = 0
    # measure the computation time
    start_time = time.time()
    # number of tokens in the vocabulary
    ntokens = ntokens 

    for i in range(train.text.size()[1]):
        
        batch = i
        input_ids, targets = train.text[:,i], train.target[:,i]
        input_ids = torch.tensor(input_ids.tolist()).unsqueeze(0) 
        targets = torch.tensor(targets.tolist()).unsqueeze(0)
        optimizer.zero_grad()

        # I intend this 'output' to be the final output of the Transformer-XL....
        output = model(input_ids) 
       #... to execute this line
        loss = criterion(output.view(-1, ntokens), targets)

        loss.backward()

        torch.nn.utils.clip_grad_norm_(model.parameters(), 0.5)
        optimizer.step()
```

But I don't think the line `loss = criterion(output.view(-1, ntokens), targets)` shouldn't work since the line `output = model(input_ids)` does not actually give out the final output from the model, but it rather gives out (according to the HuggingFace documentation) prediction_scores, mems, attention, etc. How can I train TransfoXLLMHeadModel on a dataset different than just WikiText103?

Thank you,
This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions.
