Since I haven’t been able to find a way to specify the features, I manipulated the CSV from pandas and then loaded it with the nlp.Dataset.from_pandas function. So now, my code looks like this:
from transformers import AutoTokenizer, AutoModelForSequenceClassification, Trainer, TrainingArguments
import nlp
import torch
import numpy as np
from sklearn.metrics import accuracy_score, precision_recall_fscore_support
import pandas as pd
from sklearn.preprocessing import LabelEncoder

model_name = "dccuchile/bert-base-spanish-wwm-cased"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForSequenceClassification.from_pretrained(model_name)

label_encoder = LabelEncoder()
train_df = pd.read_csv("spanish_train.csv", sep="\t")
val_df = pd.read_csv("spanish_val.csv", sep="\t")
train_df["LABEL"] = label_encoder.fit_transform(train_df["LABEL"])
val_df["LABEL"] = label_encoder.fit_transform(val_df["LABEL"])

train_dataset = nlp.Dataset.from_pandas(train_df)
validation_dataset = nlp.Dataset.from_pandas(val_df)

train_dataset.remove_column_("id_str")
train_dataset.rename_column_("TWEET", "tweet")
train_dataset.rename_column_("LABEL", "label")
validation_dataset.remove_column_("id_str")
validation_dataset.rename_column_("TWEET", "tweet")
validation_dataset.rename_column_("LABEL", "label")

def tokenize(batch):
    return tokenizer(batch["tweet"], padding=True, truncation=True)

train_dataset = train_dataset.map(tokenize, batched=True, batch_size=10)
validation_dataset = validation_dataset.map(tokenize, batched=True, batch_size=10)
train_dataset.set_format('torch', columns=['input_ids', 'attention_mask', 'label'])
validation_dataset.set_format('torch', columns=['input_ids', 'attention_mask', 'label'])

def compute_metrics(pred):
    labels = pred.label_ids
    preds = pred.predictions.argmax(-1)
    precision, recall, f1, _ = precision_recall_fscore_support(labels, preds, average='binary')
    acc = accuracy_score(labels, preds)
    return {
        'accuracy': acc,
        'f1': f1,
        'precision': precision,
        'recall': recall
    }

training_args = TrainingArguments(
    output_dir='./results',
    num_train_epochs=1,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=64,
    warmup_steps=500,
    weight_decay=0.01,
    evaluate_during_training=True,
    logging_dir='./logs',
)

trainer = Trainer(
    model=model,
    args=training_args,
    compute_metrics=compute_metrics,
    train_dataset=train_dataset,
    eval_dataset=validation_dataset
)
trainer.train()

But now the error I get is the following:
/usr/local/lib/python3.6/dist-packages/nlp/utils/py_utils.py:191: UserWarning: The given NumPy array is not writeable, and PyTorch does not support non-writeable tensors. This means you can write to the underlying (supposedly non-writeable) NumPy array using the tensor. You may want to copy the array to protect its data or make it writeable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at  /pytorch/torch/csrc/utils/tensor_numpy.cpp:141.)
  return function(data_struct)
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-52-3435b262f1ae> in <module>()
----> 1 trainer.train()

6 frames
/usr/local/lib/python3.6/dist-packages/transformers/trainer.py in train(self, model_path)
    490                 self._past = None
    491 
--> 492             for step, inputs in enumerate(epoch_iterator):
    493 
    494                 # Skip past any already trained steps if resuming training

/usr/local/lib/python3.6/dist-packages/tqdm/notebook.py in __iter__(self, *args, **kwargs)
    213     def __iter__(self, *args, **kwargs):
    214         try:
--> 215             for obj in super(tqdm_notebook, self).__iter__(*args, **kwargs):
    216                 # return super(tqdm...) will not catch exception
    217                 yield obj

/usr/local/lib/python3.6/dist-packages/tqdm/std.py in __iter__(self)
   1102                 fp_write=getattr(self.fp, 'write', sys.stderr.write))
   1103 
-> 1104         for obj in iterable:
   1105             yield obj
   1106             # Update and possibly print the progressbar.

/usr/local/lib/python3.6/dist-packages/torch/utils/data/dataloader.py in __next__(self)
    361 
    362     def __next__(self):
--> 363         data = self._next_data()
    364         self._num_yielded += 1
    365         if self._dataset_kind == _DatasetKind.Iterable and \

/usr/local/lib/python3.6/dist-packages/torch/utils/data/dataloader.py in _next_data(self)
    401     def _next_data(self):
    402         index = self._next_index()  # may raise StopIteration
--> 403         data = self._dataset_fetcher.fetch(index)  # may raise StopIteration
    404         if self._pin_memory:
    405             data = _utils.pin_memory.pin_memory(data)

/usr/local/lib/python3.6/dist-packages/torch/utils/data/_utils/fetch.py in fetch(self, possibly_batched_index)
     45         else:
     46             data = self.dataset[possibly_batched_index]
---> 47         return self.collate_fn(data)

/usr/local/lib/python3.6/dist-packages/transformers/data/data_collator.py in default_data_collator(features)
     59         if k not in ("label", "label_ids") and v is not None and not isinstance(v, str):
     60             if isinstance(v, torch.Tensor):
---> 61                 batch[k] = torch.stack([f[k] for f in features])
     62             else:
     63                 batch[k] = torch.tensor([f[k] for f in features], dtype=torch.long)

RuntimeError: stack expects each tensor to be equal size, but got [99] at entry 0 and [111] at entry 1

And I don’t know where is the root cause of the problem, since I am padding and truncating in the tokenization (which I guess is what the tensors error is referring to).