foleycrafter/models/specvqgan/modules/losses/vggishish/metrics.py

import logging

import numpy as np
import scipy
import torch
from sklearn.metrics import average_precision_score, roc_auc_score

logger = logging.getLogger(f'main.{__name__}')

def metrics(targets, outputs, topk=(1, 5)):
    """
    Adapted from https://github.com/hche11/VGGSound/blob/master/utils.py

    Calculate statistics including mAP, AUC, and d-prime.
        Args:
            output: 2d tensors, (dataset_size, classes_num) - before softmax
            target: 1d tensors, (dataset_size, )
            topk: tuple
        Returns:
            metric_dict: a dict of metrics
    """
    metrics_dict = dict()

    num_cls = outputs.shape[-1]

    # accuracy@k
    _, preds = torch.topk(outputs, k=max(topk), dim=1)
    correct_for_maxtopk = preds == targets.view(-1, 1).expand_as(preds)
    for k in topk:
        metrics_dict[f'accuracy_{k}'] = float(correct_for_maxtopk[:, :k].sum() / correct_for_maxtopk.shape[0])

    # avg precision, average roc_auc, and dprime
    targets = torch.nn.functional.one_hot(targets, num_classes=num_cls)

    # ids of the predicted classes (same as softmax)
    targets_pred = torch.softmax(outputs, dim=1)

    targets = targets.numpy()
    targets_pred = targets_pred.numpy()

    # one-vs-rest
    avg_p = [average_precision_score(targets[:, c], targets_pred[:, c], average=None) for c in range(num_cls)]
    try:
        roc_aucs = [roc_auc_score(targets[:, c], targets_pred[:, c], average=None) for c in range(num_cls)]
    except ValueError:
        logger.warning('Weird... Some classes never occured in targets. Do not trust the metrics.')
        roc_aucs = np.array([0.5])
        avg_p = np.array([0])

    metrics_dict['mAP'] = np.mean(avg_p)
    metrics_dict['mROCAUC'] = np.mean(roc_aucs)
    # Percent point function (ppf) (inverse of cdf — percentiles).
    metrics_dict['dprime'] = scipy.stats.norm().ppf(metrics_dict['mROCAUC']) * np.sqrt(2)

    return metrics_dict


if __name__ == '__main__':
    targets = torch.tensor([3, 3, 1, 2, 1, 0])
    outputs = torch.tensor([
        [1.2, 1.3, 1.1, 1.5],
        [1.3, 1.4, 1.0, 1.1],
        [1.5, 1.1, 1.4, 1.3],
        [1.0, 1.2, 1.4, 1.5],
        [1.2, 1.3, 1.1, 1.1],
        [1.2, 1.1, 1.1, 1.1],
    ]).float()
    metrics_dict = metrics(targets, outputs, topk=(1, 3))
    print(metrics_dict)