After extensive searching finally found the solution. Below function calculates perplexity after every epoch. FYI I have also added my training arguments.
def compute_custom_metric(pred):
    logits = torch.from_numpy(pred.predictions)
    labels = torch.from_numpy(pred.label_ids)
    loss = F.cross_entropy(logits.view(-1, tokenizer.vocab_size), labels.view(-1))
    return {'perplexity': math.exp(loss), 'calculated_loss': loss}

training_args = TrainingArguments(
    output_dir='./some_results',
    evaluation_strategy = "epoch",
    num_train_epochs=3,
    learning_rate=1e-5,
    per_device_train_batch_size=4,
    per_device_eval_batch_size=4,
    warmup_steps=500,
    weight_decay=0.01,
    logging_dir='./some_logs',
    logging_steps=logging_steps,
    seed=seed,
    fp16=True,
    eval_accumulation_steps=50,
)