I wanted to report 90, 95, 99, etc. confidence intervals on my data using PyTorch. But confidence intervals seems too important to leave my implementation untested or criticized so I wanted feedback - should be checked by at least some expert. Furthermore, I already noticed I got NaN values when my values when negative which make me think my code only works for classification (at the very least) but I also do regression. I am also surprised that using the numpy code directly actually gave me differentiable tensors…not something I was expecting.
So is this correct?:
import numpy as np
import scipy
import torch
from torch import Tensor

P_CI = {0.90: 1.64,
        0.95: 1.96,
        0.98: 2.33,
        0.99: 2.58,
        }


def mean_confidence_interval_rfs(data, confidence=0.95):
    """
    https://stackoverflow.com/a/15034143/1601580
    """
    a = 1.0 * np.array(data)
    n = len(a)
    m, se = np.mean(a), scipy.stats.sem(a)
    h = se * scipy.stats.t.ppf((1 + confidence) / 2., n - 1)
    return m, h


def mean_confidence_interval(data, confidence=0.95):
    a = 1.0 * np.array(data)
    n = len(a)
    m, se = np.mean(a), scipy.stats.sem(a)
    h = se * scipy.stats.t.ppf((1 + confidence) / 2., n - 1)
    return m, m - h, m + h


def ci(a, p=0.95):
    import numpy as np, scipy.stats as st
    st.t.interval(p, len(a) - 1, loc=np.mean(a), scale=st.sem(a))


# def ci(a, p=0.95):
#     import statsmodels.stats.api as sms
#
#     sms.DescrStatsW(a).tconfint_mean()

def compute_confidence_interval_classification(data: Tensor,
                                               by_pass_30_data_points: bool = False,
                                               p_confidence: float = 0.95
                                               ) -> Tensor:
    """
    Computes CI interval
        [B] -> [1]
    According to [1] CI the confidence interval for classification error can be calculated as follows:
        error +/- const * sqrt( (error * (1 - error)) / n)

    The values for const are provided from statistics, and common values used are:
        1.64 (90%)
        1.96 (95%)
        2.33 (98%)
        2.58 (99%)
    Assumptions:
    Use of these confidence intervals makes some assumptions that you need to ensure you can meet. They are:

    Observations in the validation data set were drawn from the domain independently (e.g. they are independent and
    identically distributed).
    At least 30 observations were used to evaluate the model.
    This is based on some statistics of sampling theory that takes calculating the error of a classifier as a binomial
    distribution, that we have sufficient observations to approximate a normal distribution for the binomial
    distribution, and that via the central limit theorem that the more observations we classify, the closer we will get
    to the true, but unknown, model skill.

    Ref:
        - computed according to: https://machinelearningmastery.com/report-classifier-performance-confidence-intervals/

    todo:
        - how does it change for other types of losses
    """
    B: int = data.size(0)
    # assert data >= 0
    assert B >= 30 and (not by_pass_30_data_points), f' Not enough data for CI calc to be valid and approximate a' \
                                                     f'normal, you have: {B=} but needed 30.'
    const: float = P_CI[p_confidence]
    error: Tensor = data.mean()
    val = torch.sqrt((error * (1 - error)) / B)
    print(val)
    ci_interval: float = const * val
    return ci_interval


def compute_confidence_interval_regression():
    """
    todo
    :return:
    """
    raise NotImplementedError


# - tests

def ci_test():
    x: Tensor = abs(torch.randn(35))
    ci_pytorch = compute_confidence_interval_classification(x)
    ci_rfs = mean_confidence_interval(x)
    print(f'{x.var()=}')
    print(f'{ci_pytorch=}')
    print(f'{ci_rfs=}')

    x: Tensor = abs(torch.randn(35, requires_grad=True))
    ci_pytorch = compute_confidence_interval_classification(x)
    ci_rfs = mean_confidence_interval(x)
    print(f'{x.var()=}')
    print(f'{ci_pytorch=}')
    print(f'{ci_rfs=}')

    x: Tensor = torch.randn(35) - 10
    ci_pytorch = compute_confidence_interval_classification(x)
    ci_rfs = mean_confidence_interval(x)
    print(f'{x.var()=}')
    print(f'{ci_pytorch=}')
    print(f'{ci_rfs=}')


if __name__ == '__main__':
    ci_test()
    print('Done, success! \a')


output:
tensor(0.0758)
x.var()=tensor(0.3983)
ci_pytorch=tensor(0.1486)
ci_rfs=(tensor(0.8259), tensor(0.5654), tensor(1.0864))
tensor(0.0796, grad_fn=<SqrtBackward>)
x.var()=tensor(0.4391, grad_fn=<VarBackward>)
ci_pytorch=tensor(0.1559, grad_fn=<MulBackward0>)
Traceback (most recent call last):
  File "/Applications/PyCharm.app/Contents/plugins/python/helpers/pydev/pydevd.py", line 1483, in _exec
    pydev_imports.execfile(file, globals, locals)  # execute the script
  File "/Applications/PyCharm.app/Contents/plugins/python/helpers/pydev/_pydev_imps/_pydev_execfile.py", line 18, in execfile
    exec(compile(contents+"\n", file, 'exec'), glob, loc)
  File "/Users/brandomiranda/ultimate-utils/ultimate-utils-proj-src/uutils/torch_uu/metrics/metrics.py", line 154, in <module>
    ci_test()
  File "/Users/brandomiranda/ultimate-utils/ultimate-utils-proj-src/uutils/torch_uu/metrics/metrics.py", line 144, in ci_test
    ci_pytorch = compute_confidence_interval_classification(x, by_pass_30_data_points)

how do I fix the code above for regression e.g. negative values of arbitrary magnitude?
Sort of surprised there isn’t an implementation already and especially not an official PyTorch one, given how important CI is supposed to be…perhaps a deep learning bad habit? Rarely seen it in papers, unfortunately.

References:

python - Compute a confidence interval from sample data - Stack Overflow
How to Report Classifier Performance with Confidence Intervals
How to generate neural network confidence intervals with Keras | by Sam Blake | HAL24K TechBlog | Medium
Variance or Confidence Interval for outputs
rfs/meta_eval.py at master · WangYueFt/rfs · GitHub
neural network - Calculate the accuracy every epoch in PyTorch - Stack Overflow
r - Construct 95% confidence interval for regression model - Stack Overflow
python - What is the proper way to compute 95% confidence intervals with PyTorch for classification and regression? - Stack Overflow