When I do that with Trainer, I have to change the line:
seq_ids = self.tokenizer(seq, truncation=True, padding='max_length', max_length=self.max_length, return_tensors='pt').to(device)

to:
seq_ids = self.tokenizer(seq, truncation=True, padding='max_length', max_length=self.max_length)

in MyDataset class and then:
def model_init():
    bert= BertForSequenceClassification.from_pretrained(model_name, num_labels=1)
    bert=nn.DataParallel(bert)
    return bert.to(device)
    #return BertForSequenceClassification.from_pretrained(model_name, num_labels=1)

training_args = TrainingArguments(
    output_dir='./results',          # output directory
    num_train_epochs=5,              # total number of training epochs
    per_device_train_batch_size=8,   # batch size per device during training
    per_device_eval_batch_size=8,   # batch size for evaluation
    do_train=True,                   # Perform training
    do_eval=True,                    # Perform evaluation
    evaluation_strategy="epoch",     # evalute after eachh epoch
    run_name="Seq Experiment"
)

from scipy.stats import pearsonr
def compute_metrics(predictions, targets):
    return pearsonr(predictions, targets)[0] ** 2

trainer = Trainer(
    model_init=model_init,                # the instantiated 🤗 Transformers model to be trained
    args=training_args,                   # training arguments, defined above
    train_dataset=train_seqs_encodings_dataset,          # training dataset
    eval_dataset=valid_seqs_encodings_dataset,             # evaluation dataset
    compute_metrics = compute_metrics,    # evaluation metrics
)

trainer.train()

I got:
RuntimeError: CUDA out of memory. Tried to allocate 512.00 MiB (GPU 0; 15.78 GiB total capacity; 11.14 GiB already allocated; 78.75 MiB free; 11.22 GiB reserved in total by PyTorch)

on 8 GPUs