image_output_1.png
from typing import List, Tuple

import matplotlib.pyplot as plt

import numpy as np

def soft_accuracy(y_true: np.ndarray, y_pred: np.ndarray) -> Tuple[List[float], float]:
    """
    Args:
        y_true (np.ndarray): GT int indices of (N, K). N is number of samples, K is number of possible answers.
        y_pred (np.ndarray): Predicted int indices of (N). N is number of samples.
    Return:
        List of scaler values for each given GT-Prediction pairs.
        Mean of above list values.
    """
    acc = []
    for yt, yp in zip(y_true, y_pred):
        ret = 0
        for k in range(len(yt)):
            res = 0
            for j in range(len(yt)):
                if k == j:
                    continue
                res += 1 if yp == yt[j] else 0
            ret += min(1, res / 3)
        ret /= len(yt)
        acc.append(ret)
    return (acc, np.mean(acc))

y_true = np.array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 1, 1, 1, 1, 1, 1, 1, 1, 2], [0, 0, 0, 1, 1, 1, 3, 3, 3, 3]])
y_pred_all_clear = np.array([0, 1, 3])
y_pred_vague_minor = np.array([0, 1, 1])
y_pred_vague_incorrect = np.array([0, 1, 2])
y_pred_normal_minor = np.array([0, 0, 3])
y_pred_normal_incorrect = np.array([0, 3, 3])
y_pred_precise_incorrect = np.array([1, 1, 3])

y_pred_vm_nm = np.array([0, 0, 1])
y_pred_vi_nm = np.array([0, 0, 2])
y_pred_vm_ni = np.array([0, 3, 1])
y_pred_vi_ni = np.array([0, 3, 2])
y_pred_vm_nm_pi = np.array([1, 0, 1])
y_pred_vi_nm_pi = np.array([1, 0, 2])
y_pred_vm_ni_pi = np.array([1, 3, 1])
y_pred_vi_ni_pi = np.array([1, 3, 2])

K = 10
scores = []
for i in range(K + 1):
    y_true_sim = np.array([[0] * i + [1] * (K - i)])
    _, score = soft_accuracy(y_true_sim, np.array([0]))
    scores.append(score)
plt.plot(scores)
plt.xlabel('number of answers which is the same as a prediction')
plt.ylabel('soft accuracy')
plt.show()