DataCleaner
// This source code is subject to the terms of the Mozilla Public License 2.0 at https://mozilla.org/MPL/2.0/
// © Electrified (electrifiedtrading)

// @version=5
// @description Functions for acquiring outlier levels and deriving a cleaned version of a series.
library("DataCleaner")

//////////////////////////////////////////////////
// @type Contains the mean (average) and the value of the standard deviation.
// @field mean The mean (average).
// @field standard The standard deviation.
export type Deviation
    float mean
    float stdev
//////////////////////////////////////////////////    

//////////////////////////////////////////////////
// @description This WMA handles NA values
wma( 
  series float src,
  simple int len) =>
    sum = 0.0
    count = 0
    last = len - 1
    for i = 0 to last
        s = src[i]
        if not na(s)
            m = last - i + 1 // triangular multiple
            count += m
            sum += s * m
    count == 0 ? na : sum/count
//////////////////////////////////////////////////

//////////////////////////////////////////////////
// @function Calculates and returns both the mean and the standard deviation.
// @param src The series to use for the calculation.
// @param len The length of the calculation.
// @param useWma When true, the calulcation is done using an improved weighted moving average to avoid giving significance to older values.
// @param smoothing The number of extra bars to help in smoothing out the result so that large spikes dont' occur from recent data.
export stdev(
  series float src,
  simple int len,
  simple bool useWma = false,
  simple int smoothing = 0) =>
    prev = src[1] // ignore current in measurment as it could throw off result.
    mean = useWma ? wma(prev, len) : ta.sma(prev, len)
    diff = prev - mean
    diff2 = diff * diff
    variance = useWma ? wma(diff2, len) : ta.sma(diff2, len)
    stdev = math.sqrt(variance)
    if smoothing > 0
        mean := ta.sma(mean, smoothing + 1)
        stdev := ta.sma(stdev, smoothing + 1)
    Deviation.new(mean, stdev)
//////////////////////////////////////////////////

//////////////////////////////////////////////////
// @function Calculates and returns both the mean and the standard deviation.
// @param src The array to use for the calculation.
export stdev(series float[] src) =>
    sum = 0.0
    count = 0
    for e in src
        if not na(e)
            sum += e
            count += 1

    for e in src
        if not na(e)
            sum += e
            count += 1

    mean = sum / count
    diff2sum = 0.0

    for e in src
        if not na(e)
            diff = e - mean
            diff2 = diff * diff
            diff2sum += diff2

    variance = diff2sum / count
    stdev = math.sqrt(variance)
    Deviation.new(mean, stdev)
//////////////////////////////////////////////////

//////////////////////////////////////////////////
// @function Gets the (standard deviation) outlier level for a given series.
// @param src The series to average and add a multiple of the standard deviation to.
// @param len The The number of bars to measure. 
// @param level The positive or negative multiple of the standard deviation to apply to the average. A positive number will be the upper boundary and a negative number will be the lower boundary. 
// @param useWma When true, the calulcation is done using an improved weighted moving average to avoid giving significance to older values.
// @param smoothing The number of extra bars to help in smoothing out the result so that large spikes dont' occur from recent data.
// @returns The average of the series plus the multiple of the standard deviation. 
export outlierLevel(
  series float src,
  simple int len,
  simple float level,
  simple bool useWma = false,
  simple int smoothing = 0) =>
    if level == 0
        runtime.error('Passing a level of (0) zero to "level" is simply the average.')
    d = stdev(src, len, useWma, smoothing)
    d.mean + d.stdev * level
//////////////////////////////////////////////////

//////////////////////////////////////////////////
// @function Returns only values that are within the maximum deviation.
// @param src The series to filter results from.
// @param len The The number of bars to measure. 
// @param maxDeviation The maximum deviation before considered an outlier. 
// @param useWma When true, the calulcation is done using an improved weighted moving average to avoid giving significance to older values.
// @param smoothing The number of extra bars to help in smoothing out the result so that large spikes dont' occur from recent data.
export naOutliers(
  series float src,
  simple int len,
  simple float maxDeviation = 4,
  simple bool useWma = false,
  simple int smoothing = 0)=>
    if maxDeviation == 0
        runtime.error('Passing a level of (0) zero to "maxDeviation" is simply the average.')
    d = stdev(src, len, useWma, smoothing)
    dev = d.stdev * maxDeviation
    src > d.mean + dev or src < d.mean - dev ? na : src
//////////////////////////////////////////////////

//////////////////////////////////////////////////
// @function Returns the source value adjusted by its standard deviation.
// @param src The series to measure.
// @param len The number of bars to measure the standard deviation.
// @param maxDeviation The maximum deviation before considered an outlier.
// @param baseline The value considered to be at center.  Typically zero.
// @param useWma When true, the calulcation is done using an improved weighted moving average to avoid giving significance to older values.
// @param smoothing The number of extra bars to help in smoothing out the result so that large spikes dont' occur from recent data.
export normalize(
  series float src,
  simple int len,
  simple int maxDeviation = 4,
  float baseline = 0,
  simple bool useWma = false,
  simple int smoothing = 0) =>
    cleaned = naOutliers(src, len, maxDeviation, useWma, smoothing)
    d = stdev(src, len, useWma, smoothing)
    src / (src >= baseline ? (d.mean + d.stdev) : math.abs(d.mean - d.stdev))
//////////////////////////////////////////////////

//////////////////////////////////////////////////
// @function Returns an array representing the result series with (outliers provided by the source) removed.
// @param src The source series to read from.
// @param result The result series.
// @param len The maximum size of the resultant array.
// @param maxDeviation The positive or negative multiple of the standard deviation to apply to the average. A positive number will be the upper boundary and a negative number will be the lower boundary. 
// @returns An array containing the cleaned series. 
export cleanUsing(
  series float src,
  series float result,
  simple int len,
  simple float maxDeviation) =>

    var cleaned = array.new_float() 

    if maxDeviation == 0
        runtime.error('Cannot clean a series with a "maxDeviation" of (0) zero.')
        cleaned

    else
        // Hold the previous value so it's not included in the set yet.
        var raw = array.new_float()

        var level = outlierLevel(src, len, maxDeviation)

        // Is the value in bounds?  Add it to the series.
        if(maxDeviation > 0 and src < level or maxDeviation < 0 and src > level)
            array.push(cleaned, result)

        // Maxed out results?
        if(array.size(cleaned) > len)
            array.shift(cleaned)

        level := outlierLevel(src, len, maxDeviation)

        cleaned    
//////////////////////////////////////////////////

//////////////////////////////////////////////////
// @function Returns an array representing the source series with outliers removed.
// @param src The source series to read from.
// @param len The maximum size of the resultant array.
// @param maxDeviation The positive or negative multiple of the standard deviation to apply to the average. A positive number will be the upper boundary and a negative number will be the lower boundary. 
// @returns An array containing the cleaned series. 
export clean(
  series float src,
  simple int len,
  simple float maxDeviation) =>
    cleanUsing(src, src, len, maxDeviation)
//////////////////////////////////////////////////

//////////////////////////////////////////////////
// @function Returns an array representing the source array with outliers removed.
// @param src The source series to read from.
// @param maxDeviation The positive or negative multiple of the standard deviation to apply to the average. A positive number will be the upper boundary and a negative number will be the lower boundary. 
// @returns An array containing the cleaned series. 
export cleanArray(
  float[] src,
  simple float maxDeviation) =>
    if array.size(src)==0
        src
    else
        d = stdev(src)
        cleaned = array.new_float()
        last = array.size(src) - 1
        level = d.stdev * maxDeviation
        hi = d.mean + level
        lo = d.mean - level
        for i = 0 to last
            v = array.get(src, i)
            if lo < v and v < hi
                array.push(cleaned, v)
        cleaned 
//////////////////////////////////////////////////

//////////////////////////////////////////////////
// @function Returns an array representing the source array with outliers removed.
// @param src The array to set outliers to N/A.
// @param maxDeviation The maximum deviation before considered an outlier.
// @returns True if there were any outliers; otherwise false.
export naArrayOutliers(
  float[] src,
  simple float maxDeviation) =>
    if array.size(src)==0
        false
    else
        d = stdev(src)
        last = array.size(src) - 1
        level = d.stdev * maxDeviation
        hi = d.mean + level
        lo = d.mean - level
        bool modified = false
        for i = 0 to last
            v = array.get(src, i)
            if lo > v or v > hi
                array.set(src, i, na)
                modified := true
        modified 
//////////////////////////////////////////////////

//////////////////////////////////////////////////
// @function Gets the (standard deviation) outlier level for a given series after a single pass of removing any outliers.
// @param src The series to average and add a multiple of the standard deviation to.
// @param len The The number of bars to measure. 
// @param level The positive or negative multiple of the standard deviation to apply to the average. A positive number will be the upper boundary and a negative number will be the lower boundary. 
// @param maxDeviation The optional standard deviation level to use when cleaning the series.  The default is the value of the provided level. 
// @param useWma When true, the calulcation is done using an improved weighted moving average to avoid giving significance to older values.
// @param smoothing The number of extra bars to help in smoothing out the result so that large spikes dont' occur from recent data.
// @returns The average of the series plus the multiple of the standard deviation. 
export outlierLevelAdjusted(
  series float src,
  simple int len,
  simple float level,
  simple float maxDeviation = 0,
  simple bool useWma = false,
  simple int smoothing = 0) =>
    if level == 0
        runtime.error('Passing a level of (0) zero to "level" is simply the average.')
    cleaned = naOutliers(src, len, maxDeviation==0 ? level : maxDeviation, useWma, smoothing)
    d = stdev(cleaned, len, useWma, smoothing)
    d.mean + d.stdev * level
//////////////////////////////////////////////////

candleHeight = high - low
src = input.source(close, "Source")
source = input.bool(false, "Use Source instead of High-Low") ? src : candleHeight
len = input.int(200, "Length")
maxDev = input.float(2, "Max Deviation", minval=1)
level = input.float(4, "Max Deviation After Cleaning", minval=1)
wmaYes = input.bool(false, "Use WMA instead of SMA")
smoothing = input.int(10, "Smoothing", 0)
//plot(ta.atr(len), "ATR", color.red)

plot(candleHeight, "Candle Height", color.yellow)

before = outlierLevel(candleHeight, len, maxDev, wmaYes, smoothing)
plot(before, "Deviation Before Adjustment", color.orange)

after = outlierLevelAdjusted(candleHeight, len, level, 0, wmaYes, smoothing)
plot(after, "Deviation After Adjustment", color.green)


