I wrote a small function to calculate the BLEU score directly from predictions and targets. Right now it’s looping through all predictions and then tokens individually, so it’s pretty slow but good enough for my use case. Any suggestions for optimization are welcome.
def compute_bleu(preds, targets):
    for i in range(len(preds)):
        res_preds, res_targets = [],[]
        for p in preds[i]:
            p_dec = tokenizer.decode(p.item())
            if not p_dec in list(tokenizer.special_tokens_map.values()):
                res_preds.append(p_dec)
        for t in targets[i]:
            t_dec = tokenizer.decode(t.item())
            if not t_dec in list(tokenizer.special_tokens_map.values()):
                res_targets.append(t_dec)
        bleu_metric.add(res_preds, [res_targets])
    return bleu_metric.compute()